feat: 支持多实例客户端异常限流
This commit is contained in:
1 parent
4cd3749806
commit
2fe108d66a
18 files changed
+432
-90
No files matched your search
@@ -1,9 +1,9 @@
|
||||
import { HttpException, Injectable } from "@nestjs/common";
|
||||
import { Inject, Injectable } from "@nestjs/common";
|
||||
import {
|
||||
CLIENT_ERROR_THROTTLE_STORE,
|
||||
type ClientErrorThrottleStore,
|
||||
} from "./client-error-throttle.store.js";
|
||||
|
||||
const RATE_WINDOW_MS = 60_000;
|
||||
const RATE_LIMIT = 10;
|
||||
const DEDUPE_TTL_MS = 10 * 60_000;
|
||||
const DEDUPE_CAPACITY = 500;
|
||||
const LOG_STACK_LIMIT = 4000;
|
||||
|
||||
export interface ClientErrorReport {
|
||||
@@ -22,27 +22,16 @@ export interface ClientErrorReceipt {
|
||||
duplicate: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* 后台渲染异常去重与限流。
|
||||
* 当前生产为单实例 PM2,因此使用进程内窗口;扩成多实例前需要换成共享存储。
|
||||
*/
|
||||
@Injectable()
|
||||
export class AdminClientErrorService {
|
||||
private readonly recentErrorIds = new Map<string, number>();
|
||||
private readonly windows = new Map<string, number[]>();
|
||||
constructor(
|
||||
@Inject(CLIENT_ERROR_THROTTLE_STORE)
|
||||
private readonly throttle: ClientErrorThrottleStore,
|
||||
) {}
|
||||
|
||||
record(report: ClientErrorReport): ClientErrorReceipt {
|
||||
const now = Date.now();
|
||||
this.prune(now);
|
||||
|
||||
const dedupeKey = `${report.subject}:${report.errorId}`;
|
||||
if (this.recentErrorIds.has(dedupeKey)) {
|
||||
return { received: true, duplicate: true };
|
||||
}
|
||||
|
||||
this.assertWithinRate(report.subject, now);
|
||||
this.recentErrorIds.set(dedupeKey, now);
|
||||
this.trimDedupe();
|
||||
async record(report: ClientErrorReport): Promise<ClientErrorReceipt> {
|
||||
const result = await this.throttle.record(report);
|
||||
if (result.duplicate) return { received: true, duplicate: true };
|
||||
|
||||
console.error("admin client error", {
|
||||
errorId: report.errorId,
|
||||
@@ -57,35 +46,4 @@ export class AdminClientErrorService {
|
||||
|
||||
return { received: true, duplicate: false };
|
||||
}
|
||||
|
||||
private assertWithinRate(subject: string, now: number): void {
|
||||
const recent = (this.windows.get(subject) ?? []).filter(
|
||||
(timestamp) => now - timestamp < RATE_WINDOW_MS,
|
||||
);
|
||||
if (recent.length >= RATE_LIMIT) {
|
||||
this.windows.set(subject, recent);
|
||||
throw new HttpException("客户端错误上报过于频繁,请稍后再试", 429);
|
||||
}
|
||||
recent.push(now);
|
||||
this.windows.set(subject, recent);
|
||||
}
|
||||
|
||||
private prune(now: number): void {
|
||||
for (const [key, timestamp] of this.recentErrorIds) {
|
||||
if (now - timestamp >= DEDUPE_TTL_MS) this.recentErrorIds.delete(key);
|
||||
}
|
||||
for (const [subject, timestamps] of this.windows) {
|
||||
const recent = timestamps.filter((timestamp) => now - timestamp < RATE_WINDOW_MS);
|
||||
if (recent.length === 0) this.windows.delete(subject);
|
||||
else this.windows.set(subject, recent);
|
||||
}
|
||||
}
|
||||
|
||||
private trimDedupe(): void {
|
||||
while (this.recentErrorIds.size > DEDUPE_CAPACITY) {
|
||||
const oldest = this.recentErrorIds.keys().next();
|
||||
if (oldest.done) return;
|
||||
this.recentErrorIds.delete(oldest.value);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,9 @@ import { InfrastructureModule } from "../infrastructure.module.js";
|
||||
import { AdminBookingController } from "./admin-booking.controller.js";
|
||||
import { AdminClientErrorController } from "./admin-client-error.controller.js";
|
||||
import { AdminClientErrorService } from "./admin-client-error.service.js";
|
||||
import { createClientErrorThrottleStore } from "./client-error-throttle.factory.js";
|
||||
import { CLIENT_ERROR_THROTTLE_STORE } from "./client-error-throttle.store.js";
|
||||
import { PrismaService } from "../../prisma.service.js";
|
||||
import { AdminBrandConfigService } from "./admin-brand-config.service.js";
|
||||
import { AdminConfigController } from "./admin-config.controller.js";
|
||||
import { AdminLocalUploadController } from "./admin-local-upload.controller.js";
|
||||
@@ -47,6 +50,11 @@ import { AdminTemplatePreviewAssetsService } from "./admin-template-preview-asse
|
||||
AdminTemplatePreviewService,
|
||||
AdminTemplatePreviewAssetsService,
|
||||
TemplateOperationMetricsService,
|
||||
{
|
||||
provide: CLIENT_ERROR_THROTTLE_STORE,
|
||||
useFactory: (prisma: PrismaService) => createClientErrorThrottleStore(prisma),
|
||||
inject: [PrismaService],
|
||||
},
|
||||
AdminClientErrorService,
|
||||
],
|
||||
})
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
import type { PrismaService } from "../../prisma.service.js";
|
||||
import { PostgresClientErrorThrottleStore } from "./client-error-throttle.postgres.store.js";
|
||||
import {
|
||||
MemoryClientErrorThrottleStore,
|
||||
type ClientErrorThrottleStore,
|
||||
} from "./client-error-throttle.store.js";
|
||||
|
||||
export function createClientErrorThrottleStore(
|
||||
prisma: PrismaService,
|
||||
mode = process.env.CLIENT_ERROR_THROTTLE_STORE ?? "memory",
|
||||
): ClientErrorThrottleStore {
|
||||
switch (mode) {
|
||||
case "memory":
|
||||
return new MemoryClientErrorThrottleStore();
|
||||
case "postgres":
|
||||
return new PostgresClientErrorThrottleStore(prisma);
|
||||
default:
|
||||
throw new Error(`Unsupported CLIENT_ERROR_THROTTLE_STORE: ${mode}`);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { HttpException } from "@nestjs/common";
|
||||
import { Prisma } from "@prisma/client";
|
||||
import type { PrismaService } from "../../prisma.service.js";
|
||||
import {
|
||||
CLIENT_ERROR_DEDUPE_TTL_MS,
|
||||
CLIENT_ERROR_RATE_LIMIT,
|
||||
CLIENT_ERROR_RATE_WINDOW_MS,
|
||||
type ClientErrorThrottleReport,
|
||||
type ClientErrorThrottleResult,
|
||||
type ClientErrorThrottleStore,
|
||||
} from "./client-error-throttle.store.js";
|
||||
|
||||
/**
|
||||
* 多实例部署使用的共享存储实现。
|
||||
* 去重与限流在同一事务中执行:限流拒绝时不会留下已去重标记。
|
||||
*/
|
||||
export class PostgresClientErrorThrottleStore implements ClientErrorThrottleStore {
|
||||
private lastCleanupAt = 0;
|
||||
|
||||
constructor(private readonly prisma: PrismaService) {}
|
||||
|
||||
async record(
|
||||
report: ClientErrorThrottleReport,
|
||||
now = Date.now(),
|
||||
): Promise<ClientErrorThrottleResult> {
|
||||
const cutoff = new Date(now - CLIENT_ERROR_DEDUPE_TTL_MS);
|
||||
const windowStart = new Date(
|
||||
Math.floor(now / CLIENT_ERROR_RATE_WINDOW_MS) * CLIENT_ERROR_RATE_WINDOW_MS,
|
||||
);
|
||||
const recordedAt = new Date(now);
|
||||
let duplicate = false;
|
||||
|
||||
await this.prisma.$transaction(async (transaction) => {
|
||||
const inserted = await transaction.$queryRaw<Array<{ id: string }>>(Prisma.sql`
|
||||
INSERT INTO "ClientErrorDedupe" ("id", "subject", "errorId", "createdAt")
|
||||
VALUES (${randomUUID()}, ${report.subject}, ${report.errorId}, ${recordedAt})
|
||||
ON CONFLICT ("subject", "errorId") DO UPDATE SET
|
||||
"createdAt" = EXCLUDED."createdAt"
|
||||
WHERE "ClientErrorDedupe"."createdAt" < ${cutoff}
|
||||
RETURNING "id"
|
||||
`);
|
||||
if (!inserted.length) {
|
||||
duplicate = true;
|
||||
return;
|
||||
}
|
||||
|
||||
const rows = await transaction.$queryRaw<Array<{ count: number }>>(Prisma.sql`
|
||||
INSERT INTO "ClientErrorThrottleWindow" ("subject", "windowStart", "count", "updatedAt")
|
||||
VALUES (${report.subject}, ${windowStart}, 1, ${recordedAt})
|
||||
ON CONFLICT ("subject") DO UPDATE SET
|
||||
"count" = CASE
|
||||
WHEN EXCLUDED."windowStart" > "ClientErrorThrottleWindow"."windowStart"
|
||||
THEN 1
|
||||
WHEN EXCLUDED."windowStart" = "ClientErrorThrottleWindow"."windowStart"
|
||||
THEN LEAST("ClientErrorThrottleWindow"."count" + 1, ${CLIENT_ERROR_RATE_LIMIT + 1})
|
||||
ELSE "ClientErrorThrottleWindow"."count"
|
||||
END,
|
||||
"windowStart" = GREATEST(
|
||||
"ClientErrorThrottleWindow"."windowStart",
|
||||
EXCLUDED."windowStart"
|
||||
),
|
||||
"updatedAt" = EXCLUDED."updatedAt"
|
||||
RETURNING "count"
|
||||
`);
|
||||
if (Number(rows[0]?.count ?? 1) > CLIENT_ERROR_RATE_LIMIT) {
|
||||
throw new HttpException("客户端错误上报过于频繁,请稍后再试", 429);
|
||||
}
|
||||
});
|
||||
|
||||
await this.cleanupIfNeeded(cutoff, windowStart, now);
|
||||
return { duplicate };
|
||||
}
|
||||
|
||||
private async cleanupIfNeeded(cutoff: Date, windowStart: Date, now: number): Promise<void> {
|
||||
if (now - this.lastCleanupAt < CLIENT_ERROR_RATE_WINDOW_MS) return;
|
||||
this.lastCleanupAt = now;
|
||||
try {
|
||||
await Promise.all([
|
||||
this.prisma.clientErrorDedupe.deleteMany({ where: { createdAt: { lt: cutoff } } }),
|
||||
this.prisma.clientErrorThrottleWindow.deleteMany({
|
||||
where: {
|
||||
windowStart: {
|
||||
lt: new Date(windowStart.getTime() - CLIENT_ERROR_RATE_WINDOW_MS),
|
||||
},
|
||||
},
|
||||
}),
|
||||
]);
|
||||
} catch (error) {
|
||||
// 清理失败不能影响诊断上报本身,下一分钟会再次尝试。
|
||||
console.warn("[api] client error throttle cleanup failed", error);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
import { HttpException } from "@nestjs/common";
|
||||
|
||||
export const CLIENT_ERROR_RATE_WINDOW_MS = 60_000;
|
||||
export const CLIENT_ERROR_RATE_LIMIT = 10;
|
||||
export const CLIENT_ERROR_DEDUPE_TTL_MS = 10 * 60_000;
|
||||
export const CLIENT_ERROR_DEDUPE_CAPACITY = 500;
|
||||
export const CLIENT_ERROR_THROTTLE_STORE = Symbol("CLIENT_ERROR_THROTTLE_STORE");
|
||||
|
||||
export interface ClientErrorThrottleReport {
|
||||
subject: string;
|
||||
errorId: string;
|
||||
}
|
||||
|
||||
export interface ClientErrorThrottleResult {
|
||||
duplicate: boolean;
|
||||
}
|
||||
|
||||
export interface ClientErrorThrottleStore {
|
||||
record(report: ClientErrorThrottleReport, now?: number): Promise<ClientErrorThrottleResult>;
|
||||
}
|
||||
|
||||
/** 单实例部署使用的进程内实现;多实例必须切换为共享存储。 */
|
||||
export class MemoryClientErrorThrottleStore implements ClientErrorThrottleStore {
|
||||
private readonly recentErrorIds = new Map<string, number>();
|
||||
private readonly windows = new Map<string, number[]>();
|
||||
|
||||
async record(
|
||||
report: ClientErrorThrottleReport,
|
||||
now = Date.now(),
|
||||
): Promise<ClientErrorThrottleResult> {
|
||||
this.prune(now);
|
||||
|
||||
const dedupeKey = `${report.subject}:${report.errorId}`;
|
||||
if (this.recentErrorIds.has(dedupeKey)) return { duplicate: true };
|
||||
|
||||
this.assertWithinRate(report.subject, now);
|
||||
this.recentErrorIds.set(dedupeKey, now);
|
||||
this.trimDedupe();
|
||||
return { duplicate: false };
|
||||
}
|
||||
|
||||
private assertWithinRate(subject: string, now: number): void {
|
||||
const recent = (this.windows.get(subject) ?? []).filter(
|
||||
(timestamp) => now - timestamp < CLIENT_ERROR_RATE_WINDOW_MS,
|
||||
);
|
||||
if (recent.length >= CLIENT_ERROR_RATE_LIMIT) {
|
||||
this.windows.set(subject, recent);
|
||||
throw new HttpException("客户端错误上报过于频繁,请稍后再试", 429);
|
||||
}
|
||||
recent.push(now);
|
||||
this.windows.set(subject, recent);
|
||||
}
|
||||
|
||||
private prune(now: number): void {
|
||||
for (const [key, timestamp] of this.recentErrorIds) {
|
||||
if (now - timestamp >= CLIENT_ERROR_DEDUPE_TTL_MS) this.recentErrorIds.delete(key);
|
||||
}
|
||||
for (const [subject, timestamps] of this.windows) {
|
||||
const recent = timestamps.filter(
|
||||
(timestamp) => now - timestamp < CLIENT_ERROR_RATE_WINDOW_MS,
|
||||
);
|
||||
if (recent.length === 0) this.windows.delete(subject);
|
||||
else this.windows.set(subject, recent);
|
||||
}
|
||||
}
|
||||
|
||||
private trimDedupe(): void {
|
||||
while (this.recentErrorIds.size > CLIENT_ERROR_DEDUPE_CAPACITY) {
|
||||
const oldest = this.recentErrorIds.keys().next();
|
||||
if (oldest.done) return;
|
||||
this.recentErrorIds.delete(oldest.value);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user