feat: 支持多实例客户端异常限流

This commit is contained in:
que01 committed 2026-10-23 12:57:06 +08:00
1 parent 4cd3749806
commit 2fe108d66a
18 files changed
+432 -90

No files matched your search

@@ -0,0 +1,28 @@
-- CreateTable
CREATE TABLE "ClientErrorThrottleWindow" (
"subject" TEXT NOT NULL,
"windowStart" TIMESTAMP(3) NOT NULL,
"count" INTEGER NOT NULL,
"updatedAt" TIMESTAMP(3) NOT NULL,
CONSTRAINT "ClientErrorThrottleWindow_pkey" PRIMARY KEY ("subject")
);
-- CreateTable
CREATE TABLE "ClientErrorDedupe" (
"id" TEXT NOT NULL,
"subject" TEXT NOT NULL,
"errorId" TEXT NOT NULL,
"createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "ClientErrorDedupe_pkey" PRIMARY KEY ("id")
);
-- CreateIndex
CREATE INDEX "ClientErrorThrottleWindow_windowStart_idx" ON "ClientErrorThrottleWindow"("windowStart");
-- CreateIndex
CREATE UNIQUE INDEX "ClientErrorDedupe_subject_errorId_key" ON "ClientErrorDedupe"("subject", "errorId");
-- CreateIndex
CREATE INDEX "ClientErrorDedupe_createdAt_idx" ON "ClientErrorDedupe"("createdAt");
+19
View File
@@ -468,6 +468,25 @@ model AdminUser {
templateAuditLogs TemplatePackageAuditLog[] @relation("TemplatePackageAuditActor")
}
model ClientErrorThrottleWindow {
subject String @id
windowStart DateTime
count Int
updatedAt DateTime @updatedAt
@@index([windowStart])
}
model ClientErrorDedupe {
id String @id @default(uuid())
subject String
errorId String
createdAt DateTime @default(now())
@@unique([subject, errorId])
@@index([createdAt])
}
model TemplatePackage {
id String @id @default(uuid())
packageKey String @unique
@@ -1,9 +1,9 @@
import { HttpException, Injectable } from "@nestjs/common";
import { Inject, Injectable } from "@nestjs/common";
import {
CLIENT_ERROR_THROTTLE_STORE,
type ClientErrorThrottleStore,
} from "./client-error-throttle.store.js";
const RATE_WINDOW_MS = 60_000;
const RATE_LIMIT = 10;
const DEDUPE_TTL_MS = 10 * 60_000;
const DEDUPE_CAPACITY = 500;
const LOG_STACK_LIMIT = 4000;
export interface ClientErrorReport {
@@ -22,27 +22,16 @@ export interface ClientErrorReceipt {
duplicate: boolean;
}
/**
* 后台渲染异常去重与限流。
* 当前生产为单实例 PM2,因此使用进程内窗口;扩成多实例前需要换成共享存储。
*/
@Injectable()
export class AdminClientErrorService {
private readonly recentErrorIds = new Map<string, number>();
private readonly windows = new Map<string, number[]>();
constructor(
@Inject(CLIENT_ERROR_THROTTLE_STORE)
private readonly throttle: ClientErrorThrottleStore,
) {}
record(report: ClientErrorReport): ClientErrorReceipt {
const now = Date.now();
this.prune(now);
const dedupeKey = `${report.subject}:${report.errorId}`;
if (this.recentErrorIds.has(dedupeKey)) {
return { received: true, duplicate: true };
}
this.assertWithinRate(report.subject, now);
this.recentErrorIds.set(dedupeKey, now);
this.trimDedupe();
async record(report: ClientErrorReport): Promise<ClientErrorReceipt> {
const result = await this.throttle.record(report);
if (result.duplicate) return { received: true, duplicate: true };
console.error("admin client error", {
errorId: report.errorId,
@@ -57,35 +46,4 @@ export class AdminClientErrorService {
return { received: true, duplicate: false };
}
private assertWithinRate(subject: string, now: number): void {
const recent = (this.windows.get(subject) ?? []).filter(
(timestamp) => now - timestamp < RATE_WINDOW_MS,
);
if (recent.length >= RATE_LIMIT) {
this.windows.set(subject, recent);
throw new HttpException("客户端错误上报过于频繁,请稍后再试", 429);
}
recent.push(now);
this.windows.set(subject, recent);
}
private prune(now: number): void {
for (const [key, timestamp] of this.recentErrorIds) {
if (now - timestamp >= DEDUPE_TTL_MS) this.recentErrorIds.delete(key);
}
for (const [subject, timestamps] of this.windows) {
const recent = timestamps.filter((timestamp) => now - timestamp < RATE_WINDOW_MS);
if (recent.length === 0) this.windows.delete(subject);
else this.windows.set(subject, recent);
}
}
private trimDedupe(): void {
while (this.recentErrorIds.size > DEDUPE_CAPACITY) {
const oldest = this.recentErrorIds.keys().next();
if (oldest.done) return;
this.recentErrorIds.delete(oldest.value);
}
}
}
@@ -4,6 +4,9 @@ import { InfrastructureModule } from "../infrastructure.module.js";
import { AdminBookingController } from "./admin-booking.controller.js";
import { AdminClientErrorController } from "./admin-client-error.controller.js";
import { AdminClientErrorService } from "./admin-client-error.service.js";
import { createClientErrorThrottleStore } from "./client-error-throttle.factory.js";
import { CLIENT_ERROR_THROTTLE_STORE } from "./client-error-throttle.store.js";
import { PrismaService } from "../../prisma.service.js";
import { AdminBrandConfigService } from "./admin-brand-config.service.js";
import { AdminConfigController } from "./admin-config.controller.js";
import { AdminLocalUploadController } from "./admin-local-upload.controller.js";
@@ -47,6 +50,11 @@ import { AdminTemplatePreviewAssetsService } from "./admin-template-preview-asse
AdminTemplatePreviewService,
AdminTemplatePreviewAssetsService,
TemplateOperationMetricsService,
{
provide: CLIENT_ERROR_THROTTLE_STORE,
useFactory: (prisma: PrismaService) => createClientErrorThrottleStore(prisma),
inject: [PrismaService],
},
AdminClientErrorService,
],
})
@@ -0,0 +1,20 @@
import type { PrismaService } from "../../prisma.service.js";
import { PostgresClientErrorThrottleStore } from "./client-error-throttle.postgres.store.js";
import {
MemoryClientErrorThrottleStore,
type ClientErrorThrottleStore,
} from "./client-error-throttle.store.js";
export function createClientErrorThrottleStore(
prisma: PrismaService,
mode = process.env.CLIENT_ERROR_THROTTLE_STORE ?? "memory",
): ClientErrorThrottleStore {
switch (mode) {
case "memory":
return new MemoryClientErrorThrottleStore();
case "postgres":
return new PostgresClientErrorThrottleStore(prisma);
default:
throw new Error(`Unsupported CLIENT_ERROR_THROTTLE_STORE: ${mode}`);
}
}
@@ -0,0 +1,94 @@
import { randomUUID } from "node:crypto";
import { HttpException } from "@nestjs/common";
import { Prisma } from "@prisma/client";
import type { PrismaService } from "../../prisma.service.js";
import {
CLIENT_ERROR_DEDUPE_TTL_MS,
CLIENT_ERROR_RATE_LIMIT,
CLIENT_ERROR_RATE_WINDOW_MS,
type ClientErrorThrottleReport,
type ClientErrorThrottleResult,
type ClientErrorThrottleStore,
} from "./client-error-throttle.store.js";
/**
* 多实例部署使用的共享存储实现。
* 去重与限流在同一事务中执行:限流拒绝时不会留下已去重标记。
*/
export class PostgresClientErrorThrottleStore implements ClientErrorThrottleStore {
private lastCleanupAt = 0;
constructor(private readonly prisma: PrismaService) {}
async record(
report: ClientErrorThrottleReport,
now = Date.now(),
): Promise<ClientErrorThrottleResult> {
const cutoff = new Date(now - CLIENT_ERROR_DEDUPE_TTL_MS);
const windowStart = new Date(
Math.floor(now / CLIENT_ERROR_RATE_WINDOW_MS) * CLIENT_ERROR_RATE_WINDOW_MS,
);
const recordedAt = new Date(now);
let duplicate = false;
await this.prisma.$transaction(async (transaction) => {
const inserted = await transaction.$queryRaw<Array<{ id: string }>>(Prisma.sql`
INSERT INTO "ClientErrorDedupe" ("id", "subject", "errorId", "createdAt")
VALUES (${randomUUID()}, ${report.subject}, ${report.errorId}, ${recordedAt})
ON CONFLICT ("subject", "errorId") DO UPDATE SET
"createdAt" = EXCLUDED."createdAt"
WHERE "ClientErrorDedupe"."createdAt" < ${cutoff}
RETURNING "id"
`);
if (!inserted.length) {
duplicate = true;
return;
}
const rows = await transaction.$queryRaw<Array<{ count: number }>>(Prisma.sql`
INSERT INTO "ClientErrorThrottleWindow" ("subject", "windowStart", "count", "updatedAt")
VALUES (${report.subject}, ${windowStart}, 1, ${recordedAt})
ON CONFLICT ("subject") DO UPDATE SET
"count" = CASE
WHEN EXCLUDED."windowStart" > "ClientErrorThrottleWindow"."windowStart"
THEN 1
WHEN EXCLUDED."windowStart" = "ClientErrorThrottleWindow"."windowStart"
THEN LEAST("ClientErrorThrottleWindow"."count" + 1, ${CLIENT_ERROR_RATE_LIMIT + 1})
ELSE "ClientErrorThrottleWindow"."count"
END,
"windowStart" = GREATEST(
"ClientErrorThrottleWindow"."windowStart",
EXCLUDED."windowStart"
),
"updatedAt" = EXCLUDED."updatedAt"
RETURNING "count"
`);
if (Number(rows[0]?.count ?? 1) > CLIENT_ERROR_RATE_LIMIT) {
throw new HttpException("客户端错误上报过于频繁,请稍后再试", 429);
}
});
await this.cleanupIfNeeded(cutoff, windowStart, now);
return { duplicate };
}
private async cleanupIfNeeded(cutoff: Date, windowStart: Date, now: number): Promise<void> {
if (now - this.lastCleanupAt < CLIENT_ERROR_RATE_WINDOW_MS) return;
this.lastCleanupAt = now;
try {
await Promise.all([
this.prisma.clientErrorDedupe.deleteMany({ where: { createdAt: { lt: cutoff } } }),
this.prisma.clientErrorThrottleWindow.deleteMany({
where: {
windowStart: {
lt: new Date(windowStart.getTime() - CLIENT_ERROR_RATE_WINDOW_MS),
},
},
}),
]);
} catch (error) {
// 清理失败不能影响诊断上报本身,下一分钟会再次尝试。
console.warn("[api] client error throttle cleanup failed", error);
}
}
}
@@ -0,0 +1,74 @@
import { HttpException } from "@nestjs/common";
export const CLIENT_ERROR_RATE_WINDOW_MS = 60_000;
export const CLIENT_ERROR_RATE_LIMIT = 10;
export const CLIENT_ERROR_DEDUPE_TTL_MS = 10 * 60_000;
export const CLIENT_ERROR_DEDUPE_CAPACITY = 500;
export const CLIENT_ERROR_THROTTLE_STORE = Symbol("CLIENT_ERROR_THROTTLE_STORE");
export interface ClientErrorThrottleReport {
subject: string;
errorId: string;
}
export interface ClientErrorThrottleResult {
duplicate: boolean;
}
export interface ClientErrorThrottleStore {
record(report: ClientErrorThrottleReport, now?: number): Promise<ClientErrorThrottleResult>;
}
/** 单实例部署使用的进程内实现;多实例必须切换为共享存储。 */
export class MemoryClientErrorThrottleStore implements ClientErrorThrottleStore {
private readonly recentErrorIds = new Map<string, number>();
private readonly windows = new Map<string, number[]>();
async record(
report: ClientErrorThrottleReport,
now = Date.now(),
): Promise<ClientErrorThrottleResult> {
this.prune(now);
const dedupeKey = `${report.subject}:${report.errorId}`;
if (this.recentErrorIds.has(dedupeKey)) return { duplicate: true };
this.assertWithinRate(report.subject, now);
this.recentErrorIds.set(dedupeKey, now);
this.trimDedupe();
return { duplicate: false };
}
private assertWithinRate(subject: string, now: number): void {
const recent = (this.windows.get(subject) ?? []).filter(
(timestamp) => now - timestamp < CLIENT_ERROR_RATE_WINDOW_MS,
);
if (recent.length >= CLIENT_ERROR_RATE_LIMIT) {
this.windows.set(subject, recent);
throw new HttpException("客户端错误上报过于频繁,请稍后再试", 429);
}
recent.push(now);
this.windows.set(subject, recent);
}
private prune(now: number): void {
for (const [key, timestamp] of this.recentErrorIds) {
if (now - timestamp >= CLIENT_ERROR_DEDUPE_TTL_MS) this.recentErrorIds.delete(key);
}
for (const [subject, timestamps] of this.windows) {
const recent = timestamps.filter(
(timestamp) => now - timestamp < CLIENT_ERROR_RATE_WINDOW_MS,
);
if (recent.length === 0) this.windows.delete(subject);
else this.windows.set(subject, recent);
}
}
private trimDedupe(): void {
while (this.recentErrorIds.size > CLIENT_ERROR_DEDUPE_CAPACITY) {
const oldest = this.recentErrorIds.keys().next();
if (oldest.done) return;
this.recentErrorIds.delete(oldest.value);
}
}
}
@@ -2,6 +2,7 @@ import assert from "node:assert/strict";
import { test } from "node:test";
import { HttpException } from "@nestjs/common";
import { AdminClientErrorService } from "../src/modules/admin/admin-client-error.service.js";
import { MemoryClientErrorThrottleStore } from "../src/modules/admin/client-error-throttle.store.js";
function report(errorId: string, subject = "admin:user-1") {
return {
@@ -15,40 +16,42 @@ function report(errorId: string, subject = "admin:user-1") {
};
}
test("首次上报记录,相同 errorId 在窗口内按管理员去重", () => {
const service = new AdminClientErrorService();
assert.deepEqual(service.record(report("admin-error-1")), {
function service() {
return new AdminClientErrorService(new MemoryClientErrorThrottleStore());
}
test("首次上报记录,相同 errorId 在窗口内按管理员去重", async () => {
const errors = service();
assert.deepEqual(await errors.record(report("admin-error-1")), {
received: true,
duplicate: false,
});
assert.deepEqual(service.record(report("admin-error-1")), {
assert.deepEqual(await errors.record(report("admin-error-1")), {
received: true,
duplicate: true,
});
// 不同管理员使用相同 errorId 时互不影响
assert.deepEqual(service.record(report("admin-error-1", "admin:user-2")), {
assert.deepEqual(await errors.record(report("admin-error-1", "admin:user-2")), {
received: true,
duplicate: false,
});
});
test("同一管理员 60 秒内最多接受 10 条不同错误", () => {
const service = new AdminClientErrorService();
test("同一管理员 60 秒内最多接受 10 条不同错误", async () => {
const errors = service();
for (let index = 0; index < 10; index += 1) {
assert.deepEqual(service.record(report(`admin-rate-${index}`)), {
assert.deepEqual(await errors.record(report(`admin-rate-${index}`)), {
received: true,
duplicate: false,
});
}
try {
service.record(report("admin-rate-overflow"));
assert.fail("应当触发限流");
} catch (error) {
await assert.rejects(errors.record(report("admin-rate-overflow")), (error) => {
assert.ok(error instanceof HttpException);
assert.equal(error.getStatus(), 429);
}
return true;
});
// 另一个管理员不受影响
assert.deepEqual(service.record(report("admin-rate-other", "admin:user-2")), {
assert.deepEqual(await errors.record(report("admin-rate-other", "admin:user-2")), {
received: true,
duplicate: false,
});
@@ -0,0 +1,24 @@
import assert from "node:assert/strict";
import { test } from "node:test";
import { createClientErrorThrottleStore } from "../src/modules/admin/client-error-throttle.factory.js";
import { PostgresClientErrorThrottleStore } from "../src/modules/admin/client-error-throttle.postgres.store.js";
import { MemoryClientErrorThrottleStore } from "../src/modules/admin/client-error-throttle.store.js";
const prisma = {} as never;
test("默认使用进程内存储", () => {
const store = createClientErrorThrottleStore(prisma, "memory");
assert.ok(store instanceof MemoryClientErrorThrottleStore);
});
test("多实例可切换为 Postgres 共享存储", () => {
const store = createClientErrorThrottleStore(prisma, "postgres");
assert.ok(store instanceof PostgresClientErrorThrottleStore);
});
test("未知存储模式在启动时快速失败", () => {
assert.throws(
() => createClientErrorThrottleStore(prisma, "redis"),
/Unsupported CLIENT_ERROR_THROTTLE_STORE/,
);
});
+87
View File
@@ -12,6 +12,11 @@ import { createApp } from "../dist/main.js";
import { readIdentity, signIdentity } from "../dist/auth.js";
import { WechatAuthService } from "../dist/wechat-auth.js";
import { AdminMiniAppReleaseService } from "../dist/modules/admin/admin-miniapp-release.service.js";
import { PostgresClientErrorThrottleStore } from "../dist/modules/admin/client-error-throttle.postgres.store.js";
import {
CLIENT_ERROR_DEDUPE_TTL_MS,
CLIENT_ERROR_RATE_WINDOW_MS,
} from "../dist/modules/admin/client-error-throttle.store.js";
let app;
const prisma = new PrismaClient();
@@ -1928,6 +1933,88 @@ test("admin client error reports are authenticated, validated, and deduplicated"
assert.equal(invalid.statusCode, 400);
});
test("postgres client error store deduplicates and rate limits across instances", async () => {
const storeA = new PostgresClientErrorThrottleStore(prisma);
const storeB = new PostgresClientErrorThrottleStore(prisma);
const subjectPrefix = `pg-client-error-${randomUUID()}`;
const dedupeSubject = `${subjectPrefix}-dedupe`;
const concurrentSubject = `${subjectPrefix}-concurrent`;
const rateSubject = `${subjectPrefix}-rate`;
const lateSubject = `${subjectPrefix}-late`;
const now = Date.now();
try {
assert.deepEqual(await storeA.record({ subject: dedupeSubject, errorId: "same" }, now), {
duplicate: false,
});
assert.deepEqual(await storeB.record({ subject: dedupeSubject, errorId: "same" }, now), {
duplicate: true,
});
assert.deepEqual(
await storeB.record(
{ subject: dedupeSubject, errorId: "same" },
now + CLIENT_ERROR_DEDUPE_TTL_MS + 1_000,
),
{ duplicate: false },
);
const concurrent = await Promise.all(
Array.from({ length: 6 }, (_, index) =>
(index % 2 === 0 ? storeA : storeB).record(
{ subject: concurrentSubject, errorId: "concurrent-same" },
now,
),
),
);
assert.equal(concurrent.filter((result) => !result.duplicate).length, 1);
for (let index = 0; index < 10; index += 1) {
const store = index % 2 === 0 ? storeA : storeB;
assert.deepEqual(
await store.record({ subject: rateSubject, errorId: `rate-${index}` }, now),
{ duplicate: false },
);
}
await assert.rejects(
storeB.record({ subject: rateSubject, errorId: "rate-overflow" }, now),
(error) => error.getStatus?.() === 429,
);
assert.deepEqual(
await storeA.record(
{ subject: rateSubject, errorId: "rate-next-window" },
now + CLIENT_ERROR_RATE_WINDOW_MS,
),
{ duplicate: false },
);
// 旧窗口的迟到请求不得把窗口指针回拨,否则会额外放行一整窗口。
await storeA.record(
{ subject: lateSubject, errorId: "late-first" },
now + CLIENT_ERROR_RATE_WINDOW_MS,
);
await storeB.record({ subject: lateSubject, errorId: "late-old" }, now);
for (let index = 0; index < 9; index += 1) {
assert.deepEqual(
await storeA.record(
{ subject: lateSubject, errorId: `late-next-${index}` },
now + CLIENT_ERROR_RATE_WINDOW_MS,
),
{ duplicate: false },
);
}
await assert.rejects(
storeB.record(
{ subject: lateSubject, errorId: "late-overflow" },
now + CLIENT_ERROR_RATE_WINDOW_MS,
),
(error) => error.getStatus?.() === 429,
);
} finally {
const subjects = [dedupeSubject, concurrentSubject, rateSubject, lateSubject];
await prisma.clientErrorDedupe.deleteMany({ where: { subject: { in: subjects } } });
await prisma.clientErrorThrottleWindow.deleteMany({ where: { subject: { in: subjects } } });
}
});
async function assertProtectedRouteBoundary(url, options = {}) {
const { method = "GET", payload, forbiddenRole = "customer", headers = {} } = options;
const requestOptions = payload === undefined ? { method, headers } : { method, headers, payload };
@@ -45,7 +45,6 @@ async function main() {
.filter((entry) => entry.isDirectory() && entry.name !== targetMigration)
.map((entry) => entry.name)
.sort();
assert.equal(migrationDirs.at(-1), "20261019100000_remove_brand_profile_defaults");
await admin.$executeRawUnsafe(`CREATE SCHEMA "${schema}"`);
createdSchema = true;