feat: 支持多实例客户端异常限流

This commit is contained in:
que01 committed 2026-10-23 12:57:06 +08:00
1 parent 4cd3749806
commit 2fe108d66a
18 files changed
+432 -90

No files matched your search

@@ -1,9 +1,9 @@
import { HttpException, Injectable } from "@nestjs/common";
import { Inject, Injectable } from "@nestjs/common";
import {
CLIENT_ERROR_THROTTLE_STORE,
type ClientErrorThrottleStore,
} from "./client-error-throttle.store.js";
const RATE_WINDOW_MS = 60_000;
const RATE_LIMIT = 10;
const DEDUPE_TTL_MS = 10 * 60_000;
const DEDUPE_CAPACITY = 500;
const LOG_STACK_LIMIT = 4000;
export interface ClientErrorReport {
@@ -22,27 +22,16 @@ export interface ClientErrorReceipt {
duplicate: boolean;
}
/**
* 后台渲染异常去重与限流。
* 当前生产为单实例 PM2,因此使用进程内窗口;扩成多实例前需要换成共享存储。
*/
@Injectable()
export class AdminClientErrorService {
private readonly recentErrorIds = new Map<string, number>();
private readonly windows = new Map<string, number[]>();
constructor(
@Inject(CLIENT_ERROR_THROTTLE_STORE)
private readonly throttle: ClientErrorThrottleStore,
) {}
record(report: ClientErrorReport): ClientErrorReceipt {
const now = Date.now();
this.prune(now);
const dedupeKey = `${report.subject}:${report.errorId}`;
if (this.recentErrorIds.has(dedupeKey)) {
return { received: true, duplicate: true };
}
this.assertWithinRate(report.subject, now);
this.recentErrorIds.set(dedupeKey, now);
this.trimDedupe();
async record(report: ClientErrorReport): Promise<ClientErrorReceipt> {
const result = await this.throttle.record(report);
if (result.duplicate) return { received: true, duplicate: true };
console.error("admin client error", {
errorId: report.errorId,
@@ -57,35 +46,4 @@ export class AdminClientErrorService {
return { received: true, duplicate: false };
}
private assertWithinRate(subject: string, now: number): void {
const recent = (this.windows.get(subject) ?? []).filter(
(timestamp) => now - timestamp < RATE_WINDOW_MS,
);
if (recent.length >= RATE_LIMIT) {
this.windows.set(subject, recent);
throw new HttpException("客户端错误上报过于频繁,请稍后再试", 429);
}
recent.push(now);
this.windows.set(subject, recent);
}
private prune(now: number): void {
for (const [key, timestamp] of this.recentErrorIds) {
if (now - timestamp >= DEDUPE_TTL_MS) this.recentErrorIds.delete(key);
}
for (const [subject, timestamps] of this.windows) {
const recent = timestamps.filter((timestamp) => now - timestamp < RATE_WINDOW_MS);
if (recent.length === 0) this.windows.delete(subject);
else this.windows.set(subject, recent);
}
}
private trimDedupe(): void {
while (this.recentErrorIds.size > DEDUPE_CAPACITY) {
const oldest = this.recentErrorIds.keys().next();
if (oldest.done) return;
this.recentErrorIds.delete(oldest.value);
}
}
}
@@ -4,6 +4,9 @@ import { InfrastructureModule } from "../infrastructure.module.js";
import { AdminBookingController } from "./admin-booking.controller.js";
import { AdminClientErrorController } from "./admin-client-error.controller.js";
import { AdminClientErrorService } from "./admin-client-error.service.js";
import { createClientErrorThrottleStore } from "./client-error-throttle.factory.js";
import { CLIENT_ERROR_THROTTLE_STORE } from "./client-error-throttle.store.js";
import { PrismaService } from "../../prisma.service.js";
import { AdminBrandConfigService } from "./admin-brand-config.service.js";
import { AdminConfigController } from "./admin-config.controller.js";
import { AdminLocalUploadController } from "./admin-local-upload.controller.js";
@@ -47,6 +50,11 @@ import { AdminTemplatePreviewAssetsService } from "./admin-template-preview-asse
AdminTemplatePreviewService,
AdminTemplatePreviewAssetsService,
TemplateOperationMetricsService,
{
provide: CLIENT_ERROR_THROTTLE_STORE,
useFactory: (prisma: PrismaService) => createClientErrorThrottleStore(prisma),
inject: [PrismaService],
},
AdminClientErrorService,
],
})
@@ -0,0 +1,20 @@
import type { PrismaService } from "../../prisma.service.js";
import { PostgresClientErrorThrottleStore } from "./client-error-throttle.postgres.store.js";
import {
MemoryClientErrorThrottleStore,
type ClientErrorThrottleStore,
} from "./client-error-throttle.store.js";
export function createClientErrorThrottleStore(
prisma: PrismaService,
mode = process.env.CLIENT_ERROR_THROTTLE_STORE ?? "memory",
): ClientErrorThrottleStore {
switch (mode) {
case "memory":
return new MemoryClientErrorThrottleStore();
case "postgres":
return new PostgresClientErrorThrottleStore(prisma);
default:
throw new Error(`Unsupported CLIENT_ERROR_THROTTLE_STORE: ${mode}`);
}
}
@@ -0,0 +1,94 @@
import { randomUUID } from "node:crypto";
import { HttpException } from "@nestjs/common";
import { Prisma } from "@prisma/client";
import type { PrismaService } from "../../prisma.service.js";
import {
CLIENT_ERROR_DEDUPE_TTL_MS,
CLIENT_ERROR_RATE_LIMIT,
CLIENT_ERROR_RATE_WINDOW_MS,
type ClientErrorThrottleReport,
type ClientErrorThrottleResult,
type ClientErrorThrottleStore,
} from "./client-error-throttle.store.js";
/**
* 多实例部署使用的共享存储实现。
* 去重与限流在同一事务中执行:限流拒绝时不会留下已去重标记。
*/
export class PostgresClientErrorThrottleStore implements ClientErrorThrottleStore {
private lastCleanupAt = 0;
constructor(private readonly prisma: PrismaService) {}
async record(
report: ClientErrorThrottleReport,
now = Date.now(),
): Promise<ClientErrorThrottleResult> {
const cutoff = new Date(now - CLIENT_ERROR_DEDUPE_TTL_MS);
const windowStart = new Date(
Math.floor(now / CLIENT_ERROR_RATE_WINDOW_MS) * CLIENT_ERROR_RATE_WINDOW_MS,
);
const recordedAt = new Date(now);
let duplicate = false;
await this.prisma.$transaction(async (transaction) => {
const inserted = await transaction.$queryRaw<Array<{ id: string }>>(Prisma.sql`
INSERT INTO "ClientErrorDedupe" ("id", "subject", "errorId", "createdAt")
VALUES (${randomUUID()}, ${report.subject}, ${report.errorId}, ${recordedAt})
ON CONFLICT ("subject", "errorId") DO UPDATE SET
"createdAt" = EXCLUDED."createdAt"
WHERE "ClientErrorDedupe"."createdAt" < ${cutoff}
RETURNING "id"
`);
if (!inserted.length) {
duplicate = true;
return;
}
const rows = await transaction.$queryRaw<Array<{ count: number }>>(Prisma.sql`
INSERT INTO "ClientErrorThrottleWindow" ("subject", "windowStart", "count", "updatedAt")
VALUES (${report.subject}, ${windowStart}, 1, ${recordedAt})
ON CONFLICT ("subject") DO UPDATE SET
"count" = CASE
WHEN EXCLUDED."windowStart" > "ClientErrorThrottleWindow"."windowStart"
THEN 1
WHEN EXCLUDED."windowStart" = "ClientErrorThrottleWindow"."windowStart"
THEN LEAST("ClientErrorThrottleWindow"."count" + 1, ${CLIENT_ERROR_RATE_LIMIT + 1})
ELSE "ClientErrorThrottleWindow"."count"
END,
"windowStart" = GREATEST(
"ClientErrorThrottleWindow"."windowStart",
EXCLUDED."windowStart"
),
"updatedAt" = EXCLUDED."updatedAt"
RETURNING "count"
`);
if (Number(rows[0]?.count ?? 1) > CLIENT_ERROR_RATE_LIMIT) {
throw new HttpException("客户端错误上报过于频繁,请稍后再试", 429);
}
});
await this.cleanupIfNeeded(cutoff, windowStart, now);
return { duplicate };
}
private async cleanupIfNeeded(cutoff: Date, windowStart: Date, now: number): Promise<void> {
if (now - this.lastCleanupAt < CLIENT_ERROR_RATE_WINDOW_MS) return;
this.lastCleanupAt = now;
try {
await Promise.all([
this.prisma.clientErrorDedupe.deleteMany({ where: { createdAt: { lt: cutoff } } }),
this.prisma.clientErrorThrottleWindow.deleteMany({
where: {
windowStart: {
lt: new Date(windowStart.getTime() - CLIENT_ERROR_RATE_WINDOW_MS),
},
},
}),
]);
} catch (error) {
// 清理失败不能影响诊断上报本身,下一分钟会再次尝试。
console.warn("[api] client error throttle cleanup failed", error);
}
}
}
@@ -0,0 +1,74 @@
import { HttpException } from "@nestjs/common";
export const CLIENT_ERROR_RATE_WINDOW_MS = 60_000;
export const CLIENT_ERROR_RATE_LIMIT = 10;
export const CLIENT_ERROR_DEDUPE_TTL_MS = 10 * 60_000;
export const CLIENT_ERROR_DEDUPE_CAPACITY = 500;
export const CLIENT_ERROR_THROTTLE_STORE = Symbol("CLIENT_ERROR_THROTTLE_STORE");
export interface ClientErrorThrottleReport {
subject: string;
errorId: string;
}
export interface ClientErrorThrottleResult {
duplicate: boolean;
}
export interface ClientErrorThrottleStore {
record(report: ClientErrorThrottleReport, now?: number): Promise<ClientErrorThrottleResult>;
}
/** 单实例部署使用的进程内实现;多实例必须切换为共享存储。 */
export class MemoryClientErrorThrottleStore implements ClientErrorThrottleStore {
private readonly recentErrorIds = new Map<string, number>();
private readonly windows = new Map<string, number[]>();
async record(
report: ClientErrorThrottleReport,
now = Date.now(),
): Promise<ClientErrorThrottleResult> {
this.prune(now);
const dedupeKey = `${report.subject}:${report.errorId}`;
if (this.recentErrorIds.has(dedupeKey)) return { duplicate: true };
this.assertWithinRate(report.subject, now);
this.recentErrorIds.set(dedupeKey, now);
this.trimDedupe();
return { duplicate: false };
}
private assertWithinRate(subject: string, now: number): void {
const recent = (this.windows.get(subject) ?? []).filter(
(timestamp) => now - timestamp < CLIENT_ERROR_RATE_WINDOW_MS,
);
if (recent.length >= CLIENT_ERROR_RATE_LIMIT) {
this.windows.set(subject, recent);
throw new HttpException("客户端错误上报过于频繁,请稍后再试", 429);
}
recent.push(now);
this.windows.set(subject, recent);
}
private prune(now: number): void {
for (const [key, timestamp] of this.recentErrorIds) {
if (now - timestamp >= CLIENT_ERROR_DEDUPE_TTL_MS) this.recentErrorIds.delete(key);
}
for (const [subject, timestamps] of this.windows) {
const recent = timestamps.filter(
(timestamp) => now - timestamp < CLIENT_ERROR_RATE_WINDOW_MS,
);
if (recent.length === 0) this.windows.delete(subject);
else this.windows.set(subject, recent);
}
}
private trimDedupe(): void {
while (this.recentErrorIds.size > CLIENT_ERROR_DEDUPE_CAPACITY) {
const oldest = this.recentErrorIds.keys().next();
if (oldest.done) return;
this.recentErrorIds.delete(oldest.value);
}
}
}