From dc6e367745b8579bb6aa04f60f13e2819ae257f9 Mon Sep 17 00:00:00 2001 From: ding113 Date: Mon, 3 Aug 2026 01:57:55 +0800 Subject: [PATCH 01/18] fix(session): align queries with identity indexes --- src/repository/message.ts | 67 ++++++++++++++----- ...e-aggregate-multiple-session-stats.test.ts | 25 ++++--- .../message-session-request-query.test.ts | 6 +- .../usage-ledger/cleanup-immunity.test.ts | 25 +++---- 4 files changed, 80 insertions(+), 43 deletions(-) diff --git a/src/repository/message.ts b/src/repository/message.ts index f6a5d6b9b..48b5779a1 100644 --- a/src/repository/message.ts +++ b/src/repository/message.ts @@ -53,11 +53,7 @@ function ledgerSessionLookupForOwner(identityOrPhysicalId: string, ownerUserId?: } function ledgerCanonicalSessionLookup(identity: string, ownerUserId: number) { - const canonicalCondition = isReservedSessionIdentity(identity) - ? eq(usageLedger.sessionIdentity, identity) - : eq(ledgerSessionIdentity, identity); - - return and(canonicalCondition, eq(usageLedger.userId, ownerUserId)); + return and(eq(ledgerSessionIdentity, identity), eq(usageLedger.userId, ownerUserId)); } function messageSessionLookup(identityOrPhysicalId: string, ownerUserId?: number) { @@ -1692,7 +1688,7 @@ export async function findSessionRequestLocator( isNull(messageRequest.deletedAt) ) ) - .orderBy(desc(messageRequest.createdAt), desc(messageRequest.id)) + .orderBy(sql`${messageRequest.createdAt} DESC NULLS LAST`, desc(messageRequest.id)) .limit(1); if ( @@ -1793,17 +1789,50 @@ export async function aggregateMultipleSessionStats( affinity_fingerprint, user_agent, api_type - FROM message_request - WHERE - ( - session_identity = sid - OR session_id = sid - ) - AND deleted_at IS NULL - ${ownerCondition} + FROM ( + SELECT + id, + session_id, + session_identity, + user_id, + key, + session_identity_kind, + affinity_fingerprint, + user_agent, + api_type, + created_at, + CASE WHEN session_identity = sid THEN 0 ELSE 1 END AS identity_priority + FROM message_request + WHERE COALESCE(session_identity, session_id) = sid + AND deleted_at IS NULL + ${ownerCondition} + + UNION ALL + + SELECT + id, + session_id, + session_identity, + user_id, + key, + session_identity_kind, + affinity_fingerprint, + user_agent, + api_type, + created_at, + 1 AS identity_priority + FROM message_request + WHERE sid NOT LIKE 'pfx:%' + AND sid NOT LIKE 'sid:%' + AND session_identity IS NOT NULL + AND session_identity <> sid + AND session_id = sid + AND deleted_at IS NULL + ${ownerCondition} + ) candidates ORDER BY - CASE WHEN session_identity = sid THEN 0 ELSE 1 END, - created_at DESC, + identity_priority, + created_at DESC NULLS LAST, id DESC LIMIT 1 ) mr @@ -2349,7 +2378,9 @@ export async function findRequestsBySessionIdentity( .from(messageRequest) .where(where) .orderBy( - order === "asc" ? asc(messageRequest.createdAt) : desc(messageRequest.createdAt), + order === "asc" + ? asc(messageRequest.createdAt) + : sql`${messageRequest.createdAt} DESC NULLS LAST`, order === "asc" ? asc(messageRequest.id) : desc(messageRequest.id) ) .limit(limit) @@ -2478,7 +2509,7 @@ export async function findAdjacentSessionRequests( ) ) ) - .orderBy(desc(messageRequest.createdAt), desc(messageRequest.id)) + .orderBy(sql`${messageRequest.createdAt} DESC NULLS LAST`, desc(messageRequest.id)) .limit(1); const [next] = await db .select(selection) diff --git a/tests/unit/repository/message-aggregate-multiple-session-stats.test.ts b/tests/unit/repository/message-aggregate-multiple-session-stats.test.ts index 2894d2f78..3db308eb4 100644 --- a/tests/unit/repository/message-aggregate-multiple-session-stats.test.ts +++ b/tests/unit/repository/message-aggregate-multiple-session-stats.test.ts @@ -199,8 +199,11 @@ describe("message repository aggregateMultipleSessionStats", () => { const ownerQuery = sqlText(boundary.execute.mock.calls.at(0)?.at(0)); expect(ownerQuery).toContain("unnest"); expect(ownerQuery).toContain("session-without-owner"); - expect(ownerQuery).toContain("order by case when session_identity"); - expect(ownerQuery).toContain("created_at desc, id desc"); + expect(ownerQuery).toContain("union all"); + expect(ownerQuery).toContain("coalesce(session_identity, session_id) = sid"); + expect(ownerQuery).toContain("sid not like 'pfx:%'"); + expect(ownerQuery).toContain("sid not like 'sid:%'"); + expect(ownerQuery).toContain("created_at desc nulls last, id desc"); }); test("returns Replay-only owners with zero billing aggregates", async () => { @@ -271,11 +274,13 @@ describe("message repository aggregateMultipleSessionStats", () => { const ownerQuery = sqlText(boundary.execute.mock.calls.at(0)?.at(0)).toLowerCase(); expect(ownerQuery).toContain("session_id = sid"); expect(ownerQuery).toContain("requested_session_id"); + expect(ownerQuery).toContain("union all"); }); test("resolves a physical Session id inside the authenticated owner scope", async () => { const canonicalId = "pfx:scope:root"; - boundary.select.mockReturnValueOnce(createDrizzleQuery([statsRow(canonicalId, 1)])); + const stats = createDrizzleQuery([statsRow(canonicalId, 1)]); + boundary.select.mockReturnValueOnce(stats); boundary.selectDistinct .mockReturnValueOnce(createDrizzleQuery([])) .mockReturnValueOnce(createDrizzleQuery([])) @@ -301,6 +306,7 @@ describe("message repository aggregateMultipleSessionStats", () => { expect(ownerQuery).toContain("session_id = sid"); expect(ownerQuery).toContain("and user_id ="); expect(ownerQuery).toContain(" 7"); + expect(sqlText(stats.trace.where)).toContain("coalesce"); }); test("deduplicates current public and physical aliases using the latest identity", async () => { @@ -344,8 +350,8 @@ describe("message repository aggregateMultipleSessionStats", () => { }), ]); const ownerQuery = sqlText(boundary.execute.mock.calls.at(0)?.at(0)).toLowerCase(); - expect(ownerQuery).toContain("order by case when session_identity"); - expect(ownerQuery).toContain("created_at desc, id desc"); + expect(ownerQuery).toContain("union all"); + expect(ownerQuery).toContain("created_at desc nulls last, id desc"); }); test("prefers an exact public identity over a newer same-named physical Session", async () => { @@ -373,9 +379,10 @@ describe("message repository aggregateMultipleSessionStats", () => { await aggregateMultipleSessionStats([canonicalId]); const ownerQuery = sqlText(boundary.execute.mock.calls.at(0)?.at(0)).toLowerCase(); - expect(ownerQuery).toContain("or session_id = sid"); - expect(ownerQuery).toContain("case when session_identity"); - expect(ownerQuery).toContain("= sid then 0 else 1 end"); - expect(ownerQuery).toContain("created_at desc, id desc"); + expect(ownerQuery).toContain("union all"); + expect(ownerQuery).toContain("coalesce(session_identity, session_id) = sid"); + expect(ownerQuery).toContain("sid not like 'pfx:%'"); + expect(ownerQuery).not.toContain("session_identity = sid or session_id = sid"); + expect(ownerQuery).toContain("created_at desc nulls last, id desc"); }); }); diff --git a/tests/unit/repository/message-session-request-query.test.ts b/tests/unit/repository/message-session-request-query.test.ts index 4de3c23aa..c05a81e5c 100644 --- a/tests/unit/repository/message-session-request-query.test.ts +++ b/tests/unit/repository/message-session-request-query.test.ts @@ -213,7 +213,7 @@ describe("message repository session request queries", () => { expect(sqlText(rows.trace.where)).toContain("pfx:scope:fingerprint"); expect(sqlText(rows.trace.where)).toContain("is_replay"); expect(sqlText(rows.trace.where)).toContain("false"); - expect(sqlText(rows.trace.orderBy)).toContain("created_at desc"); + expect(sqlText(rows.trace.orderBy)).toContain("created_at desc nulls last"); expect(sqlText(rows.trace.orderBy)).toContain("id desc"); const selection = sqlText(boundary.select.mock.calls.at(1)?.at(0)).toLowerCase(); expect(selection).toContain("then row_number()"); @@ -322,6 +322,8 @@ describe("message repository session request queries", () => { expect(where).toContain("session_identity"); expect(where).toContain("session_id"); expect(where.match(/physical-a/g)).toHaveLength(2); + expect(sqlText(locator.trace.orderBy)).toContain("created_at desc nulls last"); + expect(sqlText(locator.trace.orderBy)).toContain("id desc"); }); test("filters requests without a stable selector from navigable request lists", async () => { @@ -471,7 +473,7 @@ describe("message repository session request queries", () => { expect(sqlText(previous.trace.where)).toContain("created_at"); expect(sqlText(previous.trace.where)).toContain("is_replay"); expect(sqlText(previous.trace.where)).toContain("false"); - expect(sqlText(previous.trace.orderBy)).toContain("created_at desc"); + expect(sqlText(previous.trace.orderBy)).toContain("created_at desc nulls last"); expect(sqlText(previous.trace.orderBy)).toContain("id desc"); expect(sqlText(next.trace.where)).toContain("created_at"); expect(sqlText(next.trace.where)).toContain("is_replay"); diff --git a/tests/unit/usage-ledger/cleanup-immunity.test.ts b/tests/unit/usage-ledger/cleanup-immunity.test.ts index c9a7f7657..7d3602eaf 100644 --- a/tests/unit/usage-ledger/cleanup-immunity.test.ts +++ b/tests/unit/usage-ledger/cleanup-immunity.test.ts @@ -4,6 +4,10 @@ import { describe, expect, it } from "vitest"; const serviceTs = readFileSync(resolve(process.cwd(), "src/lib/log-cleanup/service.ts"), "utf-8"); const usersTs = readFileSync(resolve(process.cwd(), "src/actions/users.ts"), "utf-8"); +const resetServiceTs = readFileSync( + resolve(process.cwd(), "src/lib/user-statistics-reset/reset-service.ts"), + "utf-8" +); describe("usage_ledger cleanup immunity", () => { it("log cleanup service never imports or queries usageLedger", () => { @@ -21,21 +25,14 @@ describe("usage_ledger cleanup immunity", () => { expect(removeUserBody).not.toContain("db.delete(usageLedger)"); }); - it("resetUserAllStatistics deletes from both tables (inside transaction)", () => { - const resetMatch = usersTs.match(/export async function resetUserAllStatistics[\s\S]*?^}/m); - expect(resetMatch).not.toBeNull(); - const resetBody = resetMatch![0]; - expect(resetBody).toContain("tx.delete(messageRequest)"); - expect(resetBody).toContain("tx.delete(usageLedger)"); + it("the dedicated statistics reset service deletes both tables in batches", () => { + expect(resetServiceTs).toContain("DELETE FROM message_request"); + expect(resetServiceTs).toContain("DELETE FROM usage_ledger"); + expect(resetServiceTs).toContain("FOR UPDATE SKIP LOCKED"); }); - it("resetUserAllStatistics is the only usageLedger delete path in users.ts", () => { - // Transaction-based: tx.delete(usageLedger) - const allDeleteMatches = [...usersTs.matchAll(/\.delete\(usageLedger\)/g)]; - expect(allDeleteMatches).toHaveLength(1); - - const deleteIndex = usersTs.indexOf(".delete(usageLedger)"); - const precedingChunk = usersTs.slice(Math.max(0, deleteIndex - 2000), deleteIndex); - expect(precedingChunk).toContain("resetUserAllStatistics"); + it("users actions enqueue the reset instead of deleting usageLedger inline", () => { + expect(usersTs).not.toContain(".delete(usageLedger)"); + expect(usersTs).toContain("enqueueUserStatisticsReset"); }); }); From 5abca8456d1ca940baf7a40579c76691b8159743 Mon Sep 17 00:00:00 2001 From: ding113 Date: Mon, 3 Aug 2026 01:58:02 +0800 Subject: [PATCH 02/18] fix(replay): bound and coordinate expired payload cleanup --- src/app/v1/_lib/proxy/replay/replay-store.ts | 115 +++++++++++++---- src/instrumentation.ts | 49 ++++---- src/lib/replay-cleanup.ts | 55 +++++++++ .../instrumentation-replay-cleanup.test.ts | 58 +++++++++ tests/unit/lib/replay-cleanup.test.ts | 116 ++++++++++++++++++ tests/unit/proxy/replay-store.test.ts | 95 +++++++++++++- 6 files changed, 437 insertions(+), 51 deletions(-) create mode 100644 src/lib/replay-cleanup.ts create mode 100644 tests/unit/instrumentation-replay-cleanup.test.ts create mode 100644 tests/unit/lib/replay-cleanup.test.ts diff --git a/src/app/v1/_lib/proxy/replay/replay-store.ts b/src/app/v1/_lib/proxy/replay/replay-store.ts index 64dc6590d..56b8f581b 100644 --- a/src/app/v1/_lib/proxy/replay/replay-store.ts +++ b/src/app/v1/_lib/proxy/replay/replay-store.ts @@ -1,6 +1,6 @@ import "server-only"; -import { and, eq, gt, lt } from "drizzle-orm"; +import { and, eq, gt, lte, sql } from "drizzle-orm"; import type Redis from "ioredis"; import { db } from "@/drizzle/db"; import { replayPayloads } from "@/drizzle/schema"; @@ -108,6 +108,40 @@ export interface ReplayPersistedRow { sourceMessageRequestId: number | null; } +export const REPLAY_CLEANUP_BATCH_SIZE = 100; + +function hasMatchingHeaders( + expected: Record, + actual: Record | null +): boolean { + if (!actual) return false; + const expectedKeys = Object.keys(expected); + const actualKeys = Object.keys(actual); + return ( + expectedKeys.length === actualKeys.length && + expectedKeys.every((key) => actual[key] === expected[key]) + ); +} + +function isMatchingPersistedReplay( + expected: ReplayPersistedRow, + actual: typeof replayPayloads.$inferSelect +): boolean { + return ( + actual.verifier === expected.verifier && + actual.scopeTag === expected.scopeTag && + actual.keyId === expected.keyId && + actual.userId === expected.userId && + actual.format === expected.format && + actual.model === expected.model && + actual.statusCode === expected.statusCode && + hasMatchingHeaders(expected.headers, actual.headersJson) && + actual.payload === expected.payload && + actual.byteSize === expected.byteSize && + actual.sourceMessageRequestId === expected.sourceMessageRequestId + ); +} + export class ReplayStore { private readonly meta: RedisKVStore; private readonly chunks: RedisListStore; @@ -302,26 +336,50 @@ export class ReplayStore { */ async persistCompleted(row: ReplayPersistedRow): Promise { const env = getEnvConfig(); - const expiresAt = new Date(Date.now() + env.REPLAY_COMPLETED_TTL_SECONDS * 1000); + const now = new Date(); + const expiresAt = new Date(now.getTime() + env.REPLAY_COMPLETED_TTL_SECONDS * 1000); + const persistedValues = { + verifier: row.verifier, + scopeTag: row.scopeTag, + keyId: row.keyId, + userId: row.userId, + format: row.format, + model: row.model, + statusCode: row.statusCode, + headersJson: row.headers, + payload: row.payload, + byteSize: row.byteSize, + sourceMessageRequestId: row.sourceMessageRequestId, + expiresAt, + }; try { - await db + const upserted = await db .insert(replayPayloads) .values({ replayId: row.replayId, - verifier: row.verifier, - scopeTag: row.scopeTag, - keyId: row.keyId, - userId: row.userId, - format: row.format, - model: row.model, - statusCode: row.statusCode, - headersJson: row.headers, - payload: row.payload, - byteSize: row.byteSize, - sourceMessageRequestId: row.sourceMessageRequestId, - expiresAt, + ...persistedValues, + }) + .onConflictDoUpdate({ + target: replayPayloads.replayId, + set: { + ...persistedValues, + createdAt: now, + }, + setWhere: lte(replayPayloads.expiresAt, now), }) - .onConflictDoNothing(); + .returning({ replayId: replayPayloads.replayId }); + + if (upserted.length > 0) return; + + const existingRows = await db + .select() + .from(replayPayloads) + .where(and(eq(replayPayloads.replayId, row.replayId), gt(replayPayloads.expiresAt, now))) + .limit(1); + const existing = existingRows[0]; + if (!existing || !isMatchingPersistedReplay(row, existing)) { + throw new Error(`durable replay conflict for ${row.replayId.slice(0, 12)}`); + } } catch (error) { logger.warn("[ReplayStore] persistCompleted failed", { error: error instanceof Error ? error.message : String(error), @@ -331,13 +389,24 @@ export class ReplayStore { } } - /** 删除 PG 持久层已过期行；返回删除数（错误由调用方处理）。 */ - async cleanupExpired(): Promise { - const deleted = await db - .delete(replayPayloads) - .where(lt(replayPayloads.expiresAt, new Date())) - .returning({ replayId: replayPayloads.replayId }); - return deleted.length; + /** 删除单批 PG 持久层过期行；返回删除数（错误由调用方处理）。 */ + async cleanupExpired(cutoff = new Date()): Promise { + const deleted = await db.execute(sql` + WITH doomed AS ( + SELECT replay_id + FROM replay_payloads + WHERE expires_at < ${cutoff} + ORDER BY expires_at, replay_id + LIMIT ${REPLAY_CLEANUP_BATCH_SIZE} + FOR UPDATE SKIP LOCKED + ) + DELETE FROM replay_payloads AS rp + USING doomed + WHERE rp.replay_id = doomed.replay_id + RETURNING 1 + `); + + return Array.isArray(deleted) ? deleted.length : 0; } async findCompleted(replayId: string): Promise { diff --git a/src/instrumentation.ts b/src/instrumentation.ts index fd2a896ac..192eb0ac1 100644 --- a/src/instrumentation.ts +++ b/src/instrumentation.ts @@ -317,32 +317,26 @@ export function describeSchedulerError(error: unknown): { }; } -/** - * F2：Replay PG 持久层过期行清理（每 10 分钟，ENABLE_REQUEST_REPLAY 开启时）。 - * 写入路径已有机会式扫尾，此任务兜底低流量期无写入的场景。 - */ -async function startReplayCleanupScheduler(): Promise { +/** Replay PG 持久层过期行清理：数据库就绪后立即执行，之后每 10 分钟执行。 */ +export async function startReplayCleanupScheduler(): Promise { if (instrumentationState.__CCH_REPLAY_CLEANUP_STARTED__) { return; } try { - const { getEnvConfig } = await import("@/lib/config/env.schema"); - if (!getEnvConfig().ENABLE_REQUEST_REPLAY) { - return; - } - const { getReplayStore } = await import("@/app/v1/_lib/proxy/replay/replay-store"); + const { runReplayCleanupTick } = await import("@/lib/replay-cleanup"); const intervalMs = 10 * 60 * 1000; - instrumentationState.__CCH_REPLAY_CLEANUP_INTERVAL_ID__ = setInterval(() => { - void getReplayStore() - .cleanupExpired() - .catch((error) => { - logger.warn("[Instrumentation] Replay cleanup tick failed", { - error: error instanceof Error ? error.message : String(error), - }); + const runTick = () => { + void runReplayCleanupTick().catch((error) => { + logger.warn("[Instrumentation] Replay cleanup tick failed", { + error: error instanceof Error ? error.message : String(error), }); - }, intervalMs); + }); + }; + + runTick(); + instrumentationState.__CCH_REPLAY_CLEANUP_INTERVAL_ID__ = setInterval(runTick, intervalMs); instrumentationState.__CCH_REPLAY_CLEANUP_STARTED__ = true; logger.info("[Instrumentation] Replay cleanup scheduler started", { @@ -573,16 +567,27 @@ export async function register() { // 初始化通知任务队列（如果启用） const { scheduleNotifications } = await import("@/lib/notification/notification-queue"); await scheduleNotifications(); + + const { startUserStatisticsResetQueue } = await import( + "@/lib/user-statistics-reset/reset-queue" + ); + startUserStatisticsResetQueue(); ( globalThis as typeof globalThis & { __CCH_STOP_BACKGROUND_QUEUES__?: () => Promise; } ).__CCH_STOP_BACKGROUND_QUEUES__ = async () => { - const [{ stopCleanupQueue }, { stopNotificationQueue }] = await Promise.all([ - import("@/lib/log-cleanup/cleanup-queue"), - import("@/lib/notification/notification-queue"), + const [{ stopCleanupQueue }, { stopNotificationQueue }, { stopUserStatisticsResetQueue }] = + await Promise.all([ + import("@/lib/log-cleanup/cleanup-queue"), + import("@/lib/notification/notification-queue"), + import("@/lib/user-statistics-reset/reset-queue"), + ]); + const results = await Promise.allSettled([ + stopCleanupQueue(), + stopNotificationQueue(), + stopUserStatisticsResetQueue(), ]); - const results = await Promise.allSettled([stopCleanupQueue(), stopNotificationQueue()]); const failures = results.flatMap((result) => result.status === "rejected" ? [result.reason] : [] ); diff --git a/src/lib/replay-cleanup.ts b/src/lib/replay-cleanup.ts new file mode 100644 index 000000000..7d3aee37b --- /dev/null +++ b/src/lib/replay-cleanup.ts @@ -0,0 +1,55 @@ +import "server-only"; + +import { getReplayStore, REPLAY_CLEANUP_BATCH_SIZE } from "@/app/v1/_lib/proxy/replay/replay-store"; +import { withAdvisoryLock } from "@/lib/migrate"; + +const REPLAY_CLEANUP_LOCK_NAME = "claude-code-hub:replay-cleanup"; +const REPLAY_CLEANUP_MAX_BATCHES = 5; +const REPLAY_CLEANUP_MAX_DURATION_MS = 30_000; + +export type ReplayCleanupTickResult = + | { status: "completed"; batches: number; deleted: number } + | { status: "skipped_locked"; batches: 0; deleted: 0 } + | { status: "skipped_running"; batches: 0; deleted: 0 }; + +let running = false; + +export async function runReplayCleanupTick(): Promise { + if (running) { + return { status: "skipped_running", batches: 0, deleted: 0 }; + } + + running = true; + try { + const locked = await withAdvisoryLock( + REPLAY_CLEANUP_LOCK_NAME, + async () => { + const cutoff = new Date(); + const startedAt = Date.now(); + let batches = 0; + let deleted = 0; + + while ( + batches < REPLAY_CLEANUP_MAX_BATCHES && + Date.now() - startedAt < REPLAY_CLEANUP_MAX_DURATION_MS + ) { + const batchDeleted = await getReplayStore().cleanupExpired(cutoff); + batches += 1; + deleted += batchDeleted; + if (batchDeleted < REPLAY_CLEANUP_BATCH_SIZE) { + break; + } + } + + return { status: "completed" as const, batches, deleted }; + }, + { skipIfLocked: true } + ); + + return locked.ran + ? (locked.result as ReplayCleanupTickResult) + : { status: "skipped_locked", batches: 0, deleted: 0 }; + } finally { + running = false; + } +} diff --git a/tests/unit/instrumentation-replay-cleanup.test.ts b/tests/unit/instrumentation-replay-cleanup.test.ts new file mode 100644 index 000000000..54bd81e77 --- /dev/null +++ b/tests/unit/instrumentation-replay-cleanup.test.ts @@ -0,0 +1,58 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const cleanupControl = vi.hoisted(() => ({ + runReplayCleanupTick: vi.fn(), +})); + +vi.mock("@/lib/logger", () => ({ + logger: { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + trace: vi.fn(), + fatal: vi.fn(), + }, +})); + +vi.mock("@/lib/replay-cleanup", () => ({ + runReplayCleanupTick: cleanupControl.runReplayCleanupTick, +})); + +import { startReplayCleanupScheduler } from "@/instrumentation"; + +describe("startReplayCleanupScheduler", () => { + beforeEach(() => { + vi.useFakeTimers(); + cleanupControl.runReplayCleanupTick.mockReset().mockResolvedValue({ + status: "completed", + batches: 1, + deleted: 0, + }); + const state = globalThis as typeof globalThis & { + __CCH_REPLAY_CLEANUP_STARTED__?: boolean; + __CCH_REPLAY_CLEANUP_INTERVAL_ID__?: ReturnType; + }; + state.__CCH_REPLAY_CLEANUP_STARTED__ = false; + state.__CCH_REPLAY_CLEANUP_INTERVAL_ID__ = undefined; + }); + + afterEach(() => { + const state = globalThis as typeof globalThis & { + __CCH_REPLAY_CLEANUP_INTERVAL_ID__?: ReturnType; + }; + if (state.__CCH_REPLAY_CLEANUP_INTERVAL_ID__) { + clearInterval(state.__CCH_REPLAY_CLEANUP_INTERVAL_ID__); + } + vi.useRealTimers(); + }); + + it("starts cleanup immediately and every ten minutes regardless of Replay enablement", async () => { + const setIntervalSpy = vi.spyOn(globalThis, "setInterval"); + await startReplayCleanupScheduler(); + await vi.runOnlyPendingTimersAsync(); + + expect(setIntervalSpy).toHaveBeenCalledWith(expect.any(Function), 10 * 60 * 1000); + expect(cleanupControl.runReplayCleanupTick).toHaveBeenCalledTimes(2); + }); +}); diff --git a/tests/unit/lib/replay-cleanup.test.ts b/tests/unit/lib/replay-cleanup.test.ts new file mode 100644 index 000000000..b40c1147e --- /dev/null +++ b/tests/unit/lib/replay-cleanup.test.ts @@ -0,0 +1,116 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const cleanupControl = vi.hoisted(() => ({ + cleanupExpired: vi.fn(), + withAdvisoryLock: vi.fn(), +})); + +vi.mock("@/app/v1/_lib/proxy/replay/replay-store", () => ({ + REPLAY_CLEANUP_BATCH_SIZE: 100, + getReplayStore: () => ({ cleanupExpired: cleanupControl.cleanupExpired }), +})); + +vi.mock("@/lib/migrate", () => ({ + withAdvisoryLock: cleanupControl.withAdvisoryLock, +})); + +import { runReplayCleanupTick } from "@/lib/replay-cleanup"; + +describe("runReplayCleanupTick", () => { + beforeEach(() => { + cleanupControl.cleanupExpired.mockReset(); + cleanupControl.withAdvisoryLock.mockReset(); + cleanupControl.withAdvisoryLock.mockImplementation( + async (_lockName: string, callback: () => Promise) => ({ + ran: true, + result: await callback(), + }) + ); + }); + + it("holds one advisory lock while deleting at most five batches", async () => { + cleanupControl.cleanupExpired.mockResolvedValue(100); + + await expect(runReplayCleanupTick()).resolves.toEqual({ + status: "completed", + batches: 5, + deleted: 500, + }); + + expect(cleanupControl.withAdvisoryLock).toHaveBeenCalledWith( + "claude-code-hub:replay-cleanup", + expect.any(Function), + { skipIfLocked: true } + ); + expect(cleanupControl.cleanupExpired).toHaveBeenCalledTimes(5); + const cutoffs = cleanupControl.cleanupExpired.mock.calls.map(([cutoff]) => cutoff); + expect(cutoffs.every((cutoff) => cutoff === cutoffs[0])).toBe(true); + }); + + it("stops after a partial batch", async () => { + cleanupControl.cleanupExpired.mockResolvedValue(42); + + await expect(runReplayCleanupTick()).resolves.toEqual({ + status: "completed", + batches: 1, + deleted: 42, + }); + expect(cleanupControl.cleanupExpired).toHaveBeenCalledTimes(1); + }); + + it("skips database work when another Pod holds the advisory lock", async () => { + cleanupControl.withAdvisoryLock.mockResolvedValue({ ran: false }); + + await expect(runReplayCleanupTick()).resolves.toEqual({ + status: "skipped_locked", + batches: 0, + deleted: 0, + }); + expect(cleanupControl.cleanupExpired).not.toHaveBeenCalled(); + }); + + it("stops before starting another batch after the 30 second budget", async () => { + cleanupControl.cleanupExpired.mockResolvedValue(100); + const now = vi.spyOn(Date, "now"); + now.mockReturnValueOnce(0).mockReturnValueOnce(0).mockReturnValue(30_000); + + await expect(runReplayCleanupTick()).resolves.toEqual({ + status: "completed", + batches: 1, + deleted: 100, + }); + now.mockRestore(); + }); + + it("skips reentrant ticks in the same process", async () => { + let releaseBatch: ((deleted: number) => void) | undefined; + cleanupControl.cleanupExpired.mockImplementation( + () => + new Promise((resolve) => { + releaseBatch = resolve; + }) + ); + + const first = runReplayCleanupTick(); + await expect(runReplayCleanupTick()).resolves.toEqual({ + status: "skipped_running", + batches: 0, + deleted: 0, + }); + + releaseBatch?.(0); + await expect(first).resolves.toEqual({ status: "completed", batches: 1, deleted: 0 }); + }); + + it("releases the in-process guard after a failed tick", async () => { + cleanupControl.cleanupExpired.mockRejectedValueOnce(new Error("database unavailable")); + await expect(runReplayCleanupTick()).rejects.toThrow("database unavailable"); + + cleanupControl.cleanupExpired.mockResolvedValueOnce(0); + await expect(runReplayCleanupTick()).resolves.toEqual({ + status: "completed", + batches: 1, + deleted: 0, + }); + }); +}); diff --git a/tests/unit/proxy/replay-store.test.ts b/tests/unit/proxy/replay-store.test.ts index 83369b272..b698894b7 100644 --- a/tests/unit/proxy/replay-store.test.ts +++ b/tests/unit/proxy/replay-store.test.ts @@ -31,7 +31,11 @@ const redisControl = vi.hoisted(() => ({ const dbState = vi.hoisted(() => ({ insertValues: [] as Record[], onConflictCalls: 0, + upsertConfigs: [] as Record[], + upsertRows: [] as Record[], deleteWheres: [] as unknown[], + executeQueries: [] as unknown[], + executeRows: [] as Record[], selectWheres: [] as unknown[], selectRows: [] as Record[], insertError: null as Error | null, @@ -71,6 +75,10 @@ vi.mock("@/lib/redis/client", () => ({ vi.mock("@/drizzle/db", () => ({ db: { + execute: async (query: unknown) => { + dbState.executeQueries.push(query); + return dbState.executeRows; + }, insert: () => ({ values: (values: Record) => { if (dbState.insertError) throw dbState.insertError; @@ -79,6 +87,11 @@ vi.mock("@/drizzle/db", () => ({ onConflictDoNothing: async () => { dbState.onConflictCalls += 1; }, + onConflictDoUpdate: (config: Record) => { + dbState.onConflictCalls += 1; + dbState.upsertConfigs.push(config); + return { returning: async () => dbState.upsertRows }; + }, }; }, }), @@ -242,7 +255,11 @@ beforeEach(() => { redisControl.client = createFakeRedis(); dbState.insertValues = []; dbState.onConflictCalls = 0; + dbState.upsertConfigs = []; + dbState.upsertRows = [{ replayId: "persisted" }]; dbState.deleteWheres = []; + dbState.executeQueries = []; + dbState.executeRows = []; dbState.selectWheres = []; dbState.selectRows = []; dbState.insertError = null; @@ -551,6 +568,63 @@ describe("ReplayStore：PG 完成持久层", () => { expect(dbState.deleteWheres).toHaveLength(0); }); + it("persistCompleted 条件替换已过期的确定性主键行", async () => { + const store = new ReplayStore(); + const row = makePersistedRow({ payload: "data: replacement\n\n" }); + + await expect(store.persistCompleted(row)).resolves.toBeUndefined(); + + expect(dbState.upsertConfigs).toHaveLength(1); + const config = dbState.upsertConfigs[0] as { + set: Record; + setWhere: unknown; + }; + expect(config.set).toMatchObject({ + verifier: row.verifier, + scopeTag: row.scopeTag, + keyId: row.keyId, + userId: row.userId, + format: row.format, + payload: row.payload, + byteSize: row.byteSize, + sourceMessageRequestId: row.sourceMessageRequestId, + }); + expect(toSqlText(config.setWhere).toLowerCase()).toContain('"expires_at" <='); + expect(dbState.selectWheres).toHaveLength(0); + }); + + it("persistCompleted 接受内容完全一致的未过期 durable 行", async () => { + dbState.upsertRows = []; + const row = makePersistedRow(); + dbState.selectRows = [ + { + ...row, + headersJson: row.headers, + expiresAt: new Date(Date.now() + 60_000), + }, + ]; + const store = new ReplayStore(); + + await expect(store.persistCompleted(row)).resolves.toBeUndefined(); + expect(dbState.selectWheres).toHaveLength(1); + }); + + it("persistCompleted 拒绝内容冲突的未过期 durable 行", async () => { + dbState.upsertRows = []; + const row = makePersistedRow(); + dbState.selectRows = [ + { + ...row, + headersJson: row.headers, + payload: "data: conflicting\n\n", + expiresAt: new Date(Date.now() + 60_000), + }, + ]; + const store = new ReplayStore(); + + await expect(store.persistCompleted(row)).rejects.toThrow("durable replay conflict"); + }); + it("persistCompleted 遇 PG 异常必须抛出（complete 屏障依赖异常走 abort）", async () => { dbState.insertError = new Error("pg down"); const store = new ReplayStore(); @@ -558,13 +632,22 @@ describe("ReplayStore：PG 完成持久层", () => { await expect(store.persistCompleted(makePersistedRow())).rejects.toThrow("pg down"); }); - it("cleanupExpired 删除过期行（供定时调度器调用）", async () => { + it("cleanupExpired 单批锁定并删除最多 100 条过期行", async () => { + dbState.executeRows = [{ deleted: 1 }, { deleted: 1 }]; const store = new ReplayStore(); - await expect(store.cleanupExpired()).resolves.toBe(0); - - expect(dbState.deleteWheres).toHaveLength(1); - const deleteSql = toSqlText(dbState.deleteWheres[0]); - expect(deleteSql).toContain('"expires_at" <'); + const cutoff = new Date("2026-08-02T12:00:00.000Z"); + + await expect(store.cleanupExpired(cutoff)).resolves.toBe(2); + + expect(dbState.executeQueries).toHaveLength(1); + const deleteSql = toSqlText(dbState.executeQueries[0]).toLowerCase(); + expect(deleteSql).toContain("with doomed as"); + expect(deleteSql).toContain("expires_at < $1"); + expect(deleteSql).toContain("order by expires_at, replay_id"); + expect(deleteSql).toContain("limit $2"); + expect(deleteSql).toContain("for update skip locked"); + expect(deleteSql).toContain("delete from replay_payloads"); + expect(deleteSql).toContain("returning 1"); }); it("findCompleted 只按 replayId + 未过期条件查询并返回首行", async () => { From 00fa1aeb9e0207c78ec52a09c057530936f114f1 Mon Sep 17 00:00:00 2001 From: ding113 Date: Mon, 3 Aug 2026 01:58:11 +0800 Subject: [PATCH 03/18] fix(dashboard): surface session and proxy status failures --- messages/en/dashboard.json | 10 +- messages/ja/dashboard.json | 10 +- messages/ru/dashboard.json | 10 +- messages/zh-CN/dashboard.json | 10 +- messages/zh-TW/dashboard.json | 10 +- .../active-sessions-client.test.tsx | 148 ++++++++++++++++++ .../_components/active-sessions-client.tsx | 55 ++++--- .../_components/active-sessions-query.test.ts | 93 +++++++++++ .../_components/active-sessions-query.ts | 56 +++++++ .../api-client/v1/actions/active-sessions.ts | 11 +- src/lib/proxy-status-tracker.ts | 31 +++- tests/unit/frontend/api-error-i18n.test.ts | 10 ++ tests/unit/lib/proxy-status-tracker.test.ts | 65 ++++++++ 13 files changed, 486 insertions(+), 33 deletions(-) create mode 100644 src/app/[locale]/dashboard/sessions/_components/active-sessions-client.test.tsx create mode 100644 src/app/[locale]/dashboard/sessions/_components/active-sessions-query.test.ts create mode 100644 src/app/[locale]/dashboard/sessions/_components/active-sessions-query.ts diff --git a/messages/en/dashboard.json b/messages/en/dashboard.json index 02fffca1e..358a19e2c 100644 --- a/messages/en/dashboard.json +++ b/messages/en/dashboard.json @@ -928,7 +928,11 @@ "storageNotEnabledHint": "Tip: Check REDIS_URL and ENABLE_RATE_LIMIT=true (session details cache). To store unredacted messages, set STORE_SESSION_MESSAGES=true." }, "errors": { - "copyFailed": "Copy Failed" + "copyFailed": "Copy Failed", + "fetchSessionsFailed": "Failed to load sessions", + "fetchSessionsTimeout": "The sessions request timed out", + "fetchSettingsFailed": "Failed to load system settings", + "retry": "Retry" }, "requestList": { "title": "Requests", @@ -1895,6 +1899,10 @@ "confirmDescription": "This will permanently delete all request logs and usage statistics for this user. This action cannot be undone.", "confirm": "Yes, Reset All", "loading": "Resetting...", + "queued": "Reset queued", + "running": "Reset in progress", + "completed": "Reset completed", + "failed": "Reset failed. You can retry.", "success": "All statistics have been reset" } }, diff --git a/messages/ja/dashboard.json b/messages/ja/dashboard.json index 2ec011c52..ee16d89c7 100644 --- a/messages/ja/dashboard.json +++ b/messages/ja/dashboard.json @@ -928,7 +928,11 @@ "storageNotEnabledHint": "ヒント: REDIS_URL と ENABLE_RATE_LIMIT=true を確認してください (セッション詳細キャッシュ)。未マスクの messages を保存するには STORE_SESSION_MESSAGES=true を設定してください。" }, "errors": { - "copyFailed": "コピー失敗" + "copyFailed": "コピー失敗", + "fetchSessionsFailed": "Session 一覧の取得に失敗しました", + "fetchSessionsTimeout": "Session 一覧の取得がタイムアウトしました", + "fetchSettingsFailed": "システム設定の取得に失敗しました", + "retry": "再試行" }, "requestList": { "title": "リクエスト一覧", @@ -1873,6 +1877,10 @@ "confirmDescription": "このユーザーのすべてのリクエストログと使用統計を完全に削除します。この操作は取り消せません。", "confirm": "はい、すべてリセット", "loading": "リセット中...", + "queued": "リセットをキューに追加しました", + "running": "統計をリセットしています", + "completed": "リセットが完了しました", + "failed": "リセットに失敗しました。再試行できます。", "success": "すべての統計がリセットされました" } }, diff --git a/messages/ru/dashboard.json b/messages/ru/dashboard.json index 14a281b3f..cbd7a6f3d 100644 --- a/messages/ru/dashboard.json +++ b/messages/ru/dashboard.json @@ -928,7 +928,11 @@ "storageNotEnabledHint": "Подсказка: проверьте REDIS_URL и ENABLE_RATE_LIMIT=true (кэш деталей сессии). Чтобы сохранять сообщения без маскировки, установите STORE_SESSION_MESSAGES=true." }, "errors": { - "copyFailed": "Не удалось скопировать" + "copyFailed": "Не удалось скопировать", + "fetchSessionsFailed": "Не удалось загрузить список Session", + "fetchSessionsTimeout": "Время ожидания списка Session истекло", + "fetchSettingsFailed": "Не удалось загрузить системные настройки", + "retry": "Повторить" }, "requestList": { "title": "Список запросов", @@ -1878,6 +1882,10 @@ "confirmDescription": "Это навсегда удалит все логи запросов и статистику использования для этого пользователя. Это действие нельзя отменить.", "confirm": "Да, сбросить все", "loading": "Сброс...", + "queued": "Сброс поставлен в очередь", + "running": "Статистика сбрасывается", + "completed": "Сброс завершен", + "failed": "Сброс не выполнен. Можно повторить.", "success": "Вся статистика сброшена" } }, diff --git a/messages/zh-CN/dashboard.json b/messages/zh-CN/dashboard.json index ea3d3885c..4578e630b 100644 --- a/messages/zh-CN/dashboard.json +++ b/messages/zh-CN/dashboard.json @@ -928,7 +928,11 @@ "storageNotEnabledHint": "提示：请检查 REDIS_URL 与 ENABLE_RATE_LIMIT=true（用于会话详情缓存）；如需保存未脱敏 messages，请设置 STORE_SESSION_MESSAGES=true。" }, "errors": { - "copyFailed": "复制失败" + "copyFailed": "复制失败", + "fetchSessionsFailed": "获取 Session 列表失败", + "fetchSessionsTimeout": "获取 Session 列表超时", + "fetchSettingsFailed": "获取系统设置失败", + "retry": "重试" }, "requestList": { "title": "请求列表", @@ -1896,6 +1900,10 @@ "confirmDescription": "这将永久删除该用户的所有请求日志和使用统计。此操作无法撤销。", "confirm": "是的，重置全部", "loading": "重置中...", + "queued": "重置任务已排队", + "running": "正在重置统计", + "completed": "重置已完成", + "failed": "重置失败，可以重试", "success": "所有统计已重置" } }, diff --git a/messages/zh-TW/dashboard.json b/messages/zh-TW/dashboard.json index 3f338fc5c..83a589fa5 100644 --- a/messages/zh-TW/dashboard.json +++ b/messages/zh-TW/dashboard.json @@ -928,7 +928,11 @@ "storageNotEnabledHint": "提示：請檢查 REDIS_URL 與 ENABLE_RATE_LIMIT=true（用於 Session 詳情快取）；如需儲存未脫敏的 messages，請設定 STORE_SESSION_MESSAGES=true。" }, "errors": { - "copyFailed": "複製失敗" + "copyFailed": "複製失敗", + "fetchSessionsFailed": "取得 Session 清單失敗", + "fetchSessionsTimeout": "取得 Session 清單逾時", + "fetchSettingsFailed": "取得系統設定失敗", + "retry": "重試" }, "requestList": { "title": "請求列表", @@ -1881,6 +1885,10 @@ "confirmDescription": "這將永久刪除該使用者的所有請求日誌和使用統計。此操作無法撤銷。", "confirm": "是的，重設全部", "loading": "重設中...", + "queued": "重設工作已排入佇列", + "running": "正在重設統計", + "completed": "重設已完成", + "failed": "重設失敗，可以重試", "success": "所有統計已重置" } }, diff --git a/src/app/[locale]/dashboard/sessions/_components/active-sessions-client.test.tsx b/src/app/[locale]/dashboard/sessions/_components/active-sessions-client.test.tsx new file mode 100644 index 000000000..3839af3c2 --- /dev/null +++ b/src/app/[locale]/dashboard/sessions/_components/active-sessions-client.test.tsx @@ -0,0 +1,148 @@ +import { act } from "react"; +import { createRoot } from "react-dom/client"; +import { NextIntlClientProvider } from "next-intl"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const queryState = vi.hoisted(() => ({ + sessions: {} as Record, + refetch: vi.fn(), +})); + +vi.mock("@tanstack/react-query", () => ({ + useQuery: (options: { queryKey: string[] }) => + options.queryKey[0] === "all-sessions" + ? queryState.sessions + : { data: { currencyDisplay: "USD" } }, +})); +vi.mock("@/i18n/routing", () => ({ useRouter: () => ({ back: vi.fn() }) })); +vi.mock("@/components/section", () => ({ + Section: ({ children }: { children: React.ReactNode }) =>
{children}
, +})); +vi.mock("./active-sessions-table", () => ({ + ActiveSessionsTable: ({ + sessions, + isLoading, + }: { + sessions: { sessionId: string }[]; + isLoading: boolean; + }) =>
{isLoading ? "loading" : sessions.map((session) => session.sessionId).join(",")}
, +})); + +import { ActiveSessionsClient } from "./active-sessions-client"; + +const messages = { + dashboard: { + sessions: { + back: "Back", + monitoring: "Sessions", + monitoringDescription: "Live status", + loadingError: "Loading failed", + refreshing: "Refreshing...", + activeSessions: "Active Sessions", + inactiveSessions: "Inactive Sessions", + pagination: { total: "total" }, + errors: { + fetchSessionsFailed: "Failed to load sessions", + fetchSessionsTimeout: "The sessions request timed out", + fetchSettingsFailed: "Failed to load settings", + retry: "Retry", + }, + }, + }, +}; + +describe("ActiveSessionsClient", () => { + let container: HTMLDivElement; + + beforeEach(() => { + ( + globalThis as typeof globalThis & { IS_REACT_ACT_ENVIRONMENT: boolean } + ).IS_REACT_ACT_ENVIRONMENT = true; + queryState.refetch.mockReset(); + container = document.createElement("div"); + document.body.appendChild(container); + }); + + afterEach(() => container.remove()); + + async function render() { + const root = createRoot(container); + await act(async () => { + root.render( + + + + ); + }); + return root; + } + + it("shows a localized timeout and an explicit retry command", async () => { + queryState.sessions = { + data: undefined, + isLoading: false, + isFetching: false, + error: new Error("FETCH_SESSIONS_TIMEOUT"), + refetch: queryState.refetch, + }; + const root = await render(); + + expect(container.textContent).toContain("The sessions request timed out"); + const retry = Array.from(container.querySelectorAll("button")).find( + (button) => button.textContent?.trim() === "Retry" + ); + await act(async () => retry?.dispatchEvent(new MouseEvent("click", { bubbles: true }))); + expect(queryState.refetch).toHaveBeenCalledTimes(1); + await act(async () => root.unmount()); + }); + + it("keeps existing rows visible during a background refresh", async () => { + queryState.sessions = { + data: { + active: [{ sessionId: "session-existing" }], + inactive: [], + totalActive: 1, + totalInactive: 0, + hasMoreActive: false, + hasMoreInactive: false, + }, + isLoading: false, + isFetching: true, + error: null, + refetch: queryState.refetch, + }; + const root = await render(); + + expect(container.textContent).toContain("session-existing"); + expect(container.textContent).toContain("Refreshing..."); + expect(container.textContent).not.toContain("loading"); + await act(async () => root.unmount()); + }); + + it("keeps existing rows visible when a background refresh fails", async () => { + queryState.sessions = { + data: { + active: [{ sessionId: "session-existing" }], + inactive: [], + totalActive: 1, + totalInactive: 0, + hasMoreActive: false, + hasMoreInactive: false, + }, + isLoading: false, + isFetching: false, + error: new Error("FETCH_SESSIONS_FAILED"), + refetch: queryState.refetch, + }; + const root = await render(); + + expect(container.textContent).toContain("session-existing"); + expect(container.textContent).toContain("Failed to load sessions"); + const retry = Array.from(container.querySelectorAll("button")).find( + (button) => button.textContent?.trim() === "Retry" + ); + await act(async () => retry?.dispatchEvent(new MouseEvent("click", { bubbles: true }))); + expect(queryState.refetch).toHaveBeenCalledTimes(1); + await act(async () => root.unmount()); + }); +}); diff --git a/src/app/[locale]/dashboard/sessions/_components/active-sessions-client.tsx b/src/app/[locale]/dashboard/sessions/_components/active-sessions-client.tsx index 14dd12312..69ddd29d0 100644 --- a/src/app/[locale]/dashboard/sessions/_components/active-sessions-client.tsx +++ b/src/app/[locale]/dashboard/sessions/_components/active-sessions-client.tsx @@ -1,29 +1,19 @@ "use client"; import { useQuery } from "@tanstack/react-query"; -import { ArrowLeft, ChevronLeft, ChevronRight } from "lucide-react"; +import { ArrowLeft, ChevronLeft, ChevronRight, RefreshCw } from "lucide-react"; import { useTranslations } from "next-intl"; import { useState } from "react"; import { Section } from "@/components/section"; import { Button } from "@/components/ui/button"; import { useRouter } from "@/i18n/routing"; -import { getAllSessions } from "@/lib/api-client/v1/actions/active-sessions"; import { getSystemSettings } from "@/lib/api-client/v1/actions/system-config"; -import type { ActiveSessionInfo } from "@/types/session"; +import { fetchAllSessionsPage, type PaginatedSessionsData } from "./active-sessions-query"; import { ActiveSessionsTable } from "./active-sessions-table"; const REFRESH_INTERVAL = 3000; // 3秒刷新一次 const PAGE_SIZE = 20; -interface PaginatedSessionsData { - active: ActiveSessionInfo[]; - inactive: ActiveSessionInfo[]; - totalActive: number; - totalInactive: number; - hasMoreActive: boolean; - hasMoreInactive: boolean; -} - /** * 活跃 Session 实时监控页面 */ @@ -35,15 +25,11 @@ export function ActiveSessionsClient() { const [activePage, setActivePage] = useState(1); const [inactivePage, setInactivePage] = useState(1); - const { data, isLoading, error, refetch } = useQuery({ + const { data, isLoading, isFetching, error, refetch } = useQuery({ queryKey: ["all-sessions", activePage, inactivePage], - queryFn: async () => { - const result = await getAllSessions(activePage, inactivePage, PAGE_SIZE); - if (!result.ok) { - throw new Error(result.error || "FETCH_SESSIONS_FAILED"); - } - return result.data; - }, + queryFn: ({ signal }) => + fetchAllSessionsPage({ activePage, inactivePage, pageSize: PAGE_SIZE, signal }), + retry: false, refetchInterval: REFRESH_INTERVAL, }); @@ -65,6 +51,9 @@ export function ActiveSessionsClient() { if (error.message === "FETCH_SESSIONS_FAILED") { return t("errors.fetchSessionsFailed"); } + if (error.message === "FETCH_SESSIONS_TIMEOUT") { + return t("errors.fetchSessionsTimeout"); + } if (error.message === "FETCH_SETTINGS_FAILED") { return t("errors.fetchSettingsFailed"); } @@ -126,14 +115,34 @@ export function ActiveSessionsClient() { - {error ? ( -
- {t("loadingError")}: {getErrorMessage(error)} + {error && !data ? ( +
+

+ {t("loadingError")}: {getErrorMessage(error)} +

+
) : ( <> + {error ? ( +
+

+ {t("loadingError")}: {getErrorMessage(error)} +

+ +
+ ) : null} {/* 活跃 Session 区域 */}
+ {isFetching && !isLoading ? ( +

{t("refreshing")}

+ ) : null} ({ getAllSessions: vi.fn() })); + +vi.mock("@/lib/api-client/v1/actions/active-sessions", () => ({ + getAllSessions: api.getAllSessions, +})); +vi.mock("@/actions/active-sessions", () => ({ + getAllSessions: api.getAllSessions, +})); + +import { fetchAllSessionsPage } from "./active-sessions-query"; + +describe("fetchAllSessionsPage", () => { + afterEach(() => { + vi.useRealTimers(); + api.getAllSessions.mockReset(); + }); + + it("fails with a stable timeout error after 15 seconds", async () => { + vi.useFakeTimers(); + api.getAllSessions.mockImplementation( + (_active: number, _inactive: number, _size: number, options: RequestInit) => + new Promise((resolve) => { + options.signal?.addEventListener("abort", () => resolve({ ok: false, error: "aborted" })); + }) + ); + const request = fetchAllSessionsPage({ + activePage: 1, + inactivePage: 1, + pageSize: 20, + signal: new AbortController().signal, + }); + const rejection = expect(request).rejects.toThrow("FETCH_SESSIONS_TIMEOUT"); + + await vi.advanceTimersByTimeAsync(15_000); + await rejection; + }); + + it("propagates caller cancellation to the browser request", async () => { + let requestSignal: AbortSignal | null | undefined; + api.getAllSessions.mockImplementation( + (_active: number, _inactive: number, _size: number, options: RequestInit) => { + requestSignal = options.signal; + return new Promise((resolve) => { + options.signal?.addEventListener("abort", () => resolve({ ok: false, error: "aborted" })); + }); + } + ); + const controller = new AbortController(); + const request = fetchAllSessionsPage({ + activePage: 1, + inactivePage: 1, + pageSize: 20, + signal: controller.signal, + }); + const rejection = expect(request).rejects.toThrow("FETCH_SESSIONS_CANCELLED"); + + controller.abort(); + await rejection; + expect(requestSignal?.aborted).toBe(true); + }); + + it("normalizes transport failures instead of exposing browser error text", async () => { + api.getAllSessions.mockRejectedValue(new TypeError("Failed to fetch")); + + await expect( + fetchAllSessionsPage({ + activePage: 1, + inactivePage: 1, + pageSize: 20, + signal: new AbortController().signal, + }) + ).rejects.toThrow("FETCH_SESSIONS_FAILED"); + }); + + it("normalizes structured backend failures", async () => { + api.getAllSessions.mockResolvedValue({ + ok: false, + error: "Bad request", + errorCode: "INVALID_FORMAT", + }); + + await expect( + fetchAllSessionsPage({ + activePage: 1, + inactivePage: 1, + pageSize: 20, + signal: new AbortController().signal, + }) + ).rejects.toThrow("FETCH_SESSIONS_FAILED"); + }); +}); diff --git a/src/app/[locale]/dashboard/sessions/_components/active-sessions-query.ts b/src/app/[locale]/dashboard/sessions/_components/active-sessions-query.ts new file mode 100644 index 000000000..7e10fa976 --- /dev/null +++ b/src/app/[locale]/dashboard/sessions/_components/active-sessions-query.ts @@ -0,0 +1,56 @@ +import { getAllSessions } from "@/lib/api-client/v1/actions/active-sessions"; +import type { ActiveSessionInfo } from "@/types/session"; + +export const SESSION_FETCH_TIMEOUT_MS = 15_000; + +export interface PaginatedSessionsData { + active: ActiveSessionInfo[]; + inactive: ActiveSessionInfo[]; + totalActive: number; + totalInactive: number; + hasMoreActive: boolean; + hasMoreInactive: boolean; +} + +export async function fetchAllSessionsPage(input: { + activePage: number; + inactivePage: number; + pageSize: number; + signal: AbortSignal; +}): Promise { + const timeoutController = new AbortController(); + const timeoutId = window.setTimeout(() => timeoutController.abort(), SESSION_FETCH_TIMEOUT_MS); + const abort = () => timeoutController.abort(); + input.signal.addEventListener("abort", abort, { once: true }); + + try { + const combinedSignal = + typeof AbortSignal.any === "function" + ? AbortSignal.any([input.signal, timeoutController.signal]) + : timeoutController.signal; + const result = await getAllSessions(input.activePage, input.inactivePage, input.pageSize, { + signal: combinedSignal, + }); + if (input.signal.aborted) { + throw new Error("FETCH_SESSIONS_CANCELLED"); + } + if (timeoutController.signal.aborted && !input.signal.aborted) { + throw new Error("FETCH_SESSIONS_TIMEOUT"); + } + if (!result.ok) { + throw new Error("FETCH_SESSIONS_FAILED"); + } + return result.data; + } catch (cause) { + if (timeoutController.signal.aborted && !input.signal.aborted) { + throw new Error("FETCH_SESSIONS_TIMEOUT"); + } + if (input.signal.aborted) { + throw cause; + } + throw new Error("FETCH_SESSIONS_FAILED"); + } finally { + window.clearTimeout(timeoutId); + input.signal.removeEventListener("abort", abort); + } +} diff --git a/src/lib/api-client/v1/actions/active-sessions.ts b/src/lib/api-client/v1/actions/active-sessions.ts index e68163925..aa0993fbd 100644 --- a/src/lib/api-client/v1/actions/active-sessions.ts +++ b/src/lib/api-client/v1/actions/active-sessions.ts @@ -1,4 +1,5 @@ import type { ActiveSessionInfo } from "@/types/session"; +import type { ApiFetchOptions } from "../fetcher"; import { apiDelete, apiGet, @@ -15,7 +16,12 @@ export function getActiveSessions() { ); } -export function getAllSessions(activePage?: number, inactivePage?: number, pageSize?: number) { +export function getAllSessions( + activePage?: number, + inactivePage?: number, + pageSize?: number, + options?: ApiFetchOptions +) { return toActionResult( apiGet( `/api/v1/sessions${searchParams({ @@ -23,7 +29,8 @@ export function getAllSessions(activePage?: number, inactivePage?: number, pageS activePage, inactivePage, pageSize, - })}` + })}`, + options ) ); } diff --git a/src/lib/proxy-status-tracker.ts b/src/lib/proxy-status-tracker.ts index 478a1a8cd..18a9f1888 100644 --- a/src/lib/proxy-status-tracker.ts +++ b/src/lib/proxy-status-tracker.ts @@ -51,6 +51,8 @@ function toTimestamp(value: Date | string | number | null | undefined): number | */ export class ProxyStatusTracker { private static instance: ProxyStatusTracker | null = null; + private cachedStatus: { value: ProxyStatusResponse; expiresAt: number } | null = null; + private inFlight: Promise | null = null; static getInstance(): ProxyStatusTracker { if (!ProxyStatusTracker.instance) { @@ -76,7 +78,27 @@ export class ProxyStatusTracker { void requestId; } - async getAllUsersStatus(): Promise { + getAllUsersStatus(): Promise { + const now = Date.now(); + if (this.cachedStatus && this.cachedStatus.expiresAt > now) { + return Promise.resolve(this.cachedStatus.value); + } + if (this.inFlight) { + return this.inFlight; + } + + this.inFlight = this.fetchAllUsersStatus() + .then((value) => { + this.cachedStatus = { value, expiresAt: Date.now() + 2_000 }; + return value; + }) + .finally(() => { + this.inFlight = null; + }); + return this.inFlight; + } + + private async fetchAllUsersStatus(): Promise { const now = Date.now(); const [dbUsers, activeRequestRows, lastRequestRows] = await Promise.all([ @@ -163,8 +185,10 @@ export class ProxyStatusTracker { .where( and( isNull(messageRequest.deletedAt), - isNull(messageRequest.durationMs), + isNull(messageRequest.statusCode), + sql`"message_request".created_at >= now() - interval '24 hours'`, eq(messageRequest.isReplay, false), + sql`("message_request".blocked_by IS NULL OR "message_request".blocked_by <> 'warmup')`, isNull(providers.deletedAt) ) ); @@ -188,8 +212,9 @@ export class ProxyStatusTracker { LEFT JOIN keys k ON k.key = mr.key AND k.deleted_at IS NULL WHERE mr.deleted_at IS NULL AND mr.is_replay = false + AND mr.status_code IS NOT NULL AND (mr.blocked_by IS NULL OR mr.blocked_by <> 'warmup') - ORDER BY mr.user_id, mr.updated_at DESC + ORDER BY mr.user_id, mr.updated_at DESC NULLS LAST, mr.id DESC `; const result = await db.execute(query); diff --git a/tests/unit/frontend/api-error-i18n.test.ts b/tests/unit/frontend/api-error-i18n.test.ts index b3349d27c..bb92af8f1 100644 --- a/tests/unit/frontend/api-error-i18n.test.ts +++ b/tests/unit/frontend/api-error-i18n.test.ts @@ -42,6 +42,16 @@ describe("v1 API error i18n mapping", () => { expect(getApiErrorMessageKey(error)).toBe("INTERNAL_ERROR"); }); + test("maps dependency outages to a retryable connection failure", () => { + const error = new ApiError({ + status: 503, + errorCode: "dependency.unavailable", + detail: "Service unavailable", + }); + + expect(getApiErrorMessageKey(error)).toBe("CONNECTION_FAILED"); + }); + test("maps provider endpoint and vendor REST codes to existing translation keys", () => { expect( getApiErrorMessageKey( diff --git a/tests/unit/lib/proxy-status-tracker.test.ts b/tests/unit/lib/proxy-status-tracker.test.ts index 911d1576a..606526e59 100644 --- a/tests/unit/lib/proxy-status-tracker.test.ts +++ b/tests/unit/lib/proxy-status-tracker.test.ts @@ -39,4 +39,69 @@ describe("ProxyStatusTracker", () => { expect(sqlText(activeQuery.trace.where[0])).toContain("is_replay = false"); expect(sqlText(boundary.execute.mock.calls[0]?.[0])).toContain("mr.is_replay = false"); }); + + it("uses a bounded active window and status_code as the active marker", async () => { + const usersQuery = createDrizzleQuery([]); + const activeQuery = createDrizzleQuery([]); + boundary.select.mockReturnValueOnce(usersQuery).mockReturnValueOnce(activeQuery); + boundary.execute.mockResolvedValueOnce([]); + + const { ProxyStatusTracker } = await import("@/lib/proxy-status-tracker"); + await ProxyStatusTracker.getInstance().getAllUsersStatus(); + + const activeWhere = sqlText(activeQuery.trace.where[0]); + expect(activeWhere).toContain("status_code is null"); + expect(activeWhere).toContain("created_at >= now() - interval '24 hours'"); + expect(activeWhere).toContain("blocked_by"); + expect(activeWhere).toContain("warmup"); + }); + + it("selects only finalized latest requests with a deterministic tie-break", async () => { + const usersQuery = createDrizzleQuery([]); + const activeQuery = createDrizzleQuery([]); + boundary.select.mockReturnValueOnce(usersQuery).mockReturnValueOnce(activeQuery); + boundary.execute.mockResolvedValueOnce([]); + + const { ProxyStatusTracker } = await import("@/lib/proxy-status-tracker"); + await ProxyStatusTracker.getInstance().getAllUsersStatus(); + + const latestSql = sqlText(boundary.execute.mock.calls[0]?.[0]); + expect(latestSql).toContain("mr.status_code is not null"); + expect(latestSql).toContain("order by mr.user_id, mr.updated_at desc nulls last, mr.id desc"); + }); + + it("coalesces concurrent calls and caches the response for two seconds", async () => { + const usersQuery = createDrizzleQuery([]); + const activeQuery = createDrizzleQuery([]); + let resolveLatest: ((rows: readonly unknown[]) => void) | undefined; + boundary.select.mockReturnValueOnce(usersQuery).mockReturnValueOnce(activeQuery); + boundary.execute.mockImplementationOnce( + () => + new Promise((resolve) => { + resolveLatest = resolve; + }) + ); + + const { ProxyStatusTracker } = await import("@/lib/proxy-status-tracker"); + const tracker = ProxyStatusTracker.getInstance(); + const first = tracker.getAllUsersStatus(); + const second = tracker.getAllUsersStatus(); + expect(first).toBe(second); + resolveLatest?.([]); + await Promise.all([first, second]); + expect(boundary.select).toHaveBeenCalledTimes(2); + expect(boundary.execute).toHaveBeenCalledTimes(1); + + await tracker.getAllUsersStatus(); + expect(boundary.select).toHaveBeenCalledTimes(2); + + vi.spyOn(Date, "now").mockReturnValue(Date.now() + 2001); + boundary.select + .mockReturnValueOnce(createDrizzleQuery([])) + .mockReturnValueOnce(createDrizzleQuery([])); + boundary.execute.mockResolvedValueOnce([]); + await tracker.getAllUsersStatus(); + expect(boundary.select).toHaveBeenCalledTimes(4); + vi.restoreAllMocks(); + }); }); From fe9b6b60091a54074539392fc3c30e1bdc0c39a5 Mon Sep 17 00:00:00 2001 From: ding113 Date: Mon, 3 Aug 2026 01:58:21 +0800 Subject: [PATCH 04/18] feat(users): queue full statistics resets --- src/actions/users.ts | 96 ++---- .../_components/user/edit-user-dialog.tsx | 82 ++++- src/app/api/v1/resources/users/handlers.ts | 35 +- src/app/api/v1/resources/users/router.ts | 39 ++- src/lib/api-client/v1/actions/users.ts | 9 +- src/lib/api-client/v1/errors.ts | 1 + src/lib/api-client/v1/openapi-types.gen.ts | 307 +++++++++++++++++- src/lib/api/v1/schemas/users.ts | 16 + src/lib/redis/cost-cache-cleanup.ts | 33 +- src/lib/user-statistics-reset/reset-queue.ts | 244 ++++++++++++++ .../user-statistics-reset/reset-service.ts | 215 ++++++++++++ .../reset-status-store.ts | 82 +++++ src/lib/user-statistics-reset/types.ts | 19 ++ tests/api/v1/users/users.test.ts | 50 ++- .../unit/actions/users-reset-5h-only.test.ts | 32 +- .../users-reset-all-statistics.test.ts | 277 ++++------------ tests/unit/api/v1/api-client-actions.test.ts | 21 ++ .../unit/lib/redis/cost-cache-cleanup.test.ts | 16 + .../lib/user-statistics-reset-queue.test.ts | 279 ++++++++++++++++ .../lib/user-statistics-reset-service.test.ts | 174 ++++++++++ ...user-statistics-reset-status-store.test.ts | 128 ++++++++ 21 files changed, 1829 insertions(+), 326 deletions(-) create mode 100644 src/lib/user-statistics-reset/reset-queue.ts create mode 100644 src/lib/user-statistics-reset/reset-service.ts create mode 100644 src/lib/user-statistics-reset/reset-status-store.ts create mode 100644 src/lib/user-statistics-reset/types.ts create mode 100644 tests/unit/lib/user-statistics-reset-queue.test.ts create mode 100644 tests/unit/lib/user-statistics-reset-service.test.ts create mode 100644 tests/unit/lib/user-statistics-reset-status-store.test.ts diff --git a/src/actions/users.ts b/src/actions/users.ts index 3bf948f4c..d5f539f61 100644 --- a/src/actions/users.ts +++ b/src/actions/users.ts @@ -1,11 +1,11 @@ "use server"; import { randomBytes } from "node:crypto"; -import { and, eq, inArray, isNull } from "drizzle-orm"; +import { and, inArray, isNull } from "drizzle-orm"; import { revalidatePath } from "next/cache"; import { getLocale, getTranslations } from "next-intl/server"; import { db } from "@/drizzle/db"; -import { messageRequest, usageLedger, users as usersTable } from "@/drizzle/schema"; +import { users as usersTable } from "@/drizzle/schema"; import { emitActionAudit } from "@/lib/audit/emit"; import { getSession } from "@/lib/auth"; import { PROVIDER_GROUP } from "@/lib/constants/provider.constants"; @@ -14,6 +14,7 @@ import { getUnauthorizedFields } from "@/lib/permissions/user-field-permissions" import { clipStartByResetAt, resolveUser5hCostResetAt } from "@/lib/rate-limit/cost-reset-utils"; import { getRedisClient } from "@/lib/redis"; import { invalidateCachedUser } from "@/lib/security/api-key-auth-cache"; +import type { UserStatisticsResetRecord } from "@/lib/user-statistics-reset/types"; import { parseDateInputAsTimezone } from "@/lib/utils/date-input"; import { ERROR_CODES } from "@/lib/utils/error-messages"; import { normalizeProviderGroup, parseProviderGroups } from "@/lib/utils/provider-group"; @@ -2338,12 +2339,15 @@ export async function resetUserLimitsOnly(userId: number): Promise } /** - * Reset ALL user statistics (logs + Redis cache + sessions) - * This is IRREVERSIBLE - deletes all messageRequest logs for the user + * Queue an irreversible reset of logs and cost caches created before the request cutoff. + * Active Session state is intentionally preserved. * * Admin only. */ -export async function resetUserAllStatistics(userId: number): Promise { +export async function resetUserAllStatistics( + userId: number +): Promise> { + let enqueueStarted = false; try { const tError = await getTranslations("errors"); @@ -2361,10 +2365,7 @@ export async function resetUserAllStatistics(userId: number): Promise k.id); - const keyHashes = keys.map((k) => k.key); const requiresRedisForFixed5h = ((user.limit5hUsd ?? 0) > 0 && (user.limit5hResetMode ?? "rolling") === "fixed") || keys.some( @@ -2372,85 +2373,32 @@ export async function resetUserAllStatistics(userId: number): Promise { - await tx.delete(messageRequest).where(eq(messageRequest.userId, userId)); - await tx.delete(usageLedger).where(eq(usageLedger.userId, userId)); - await tx - .update(usersTable) - .set({ costResetAt: null, limit5hCostResetAt: null, updatedAt: new Date() }) - .where(and(eq(usersTable.id, userId), isNull(usersTable.deletedAt))); + const { enqueueUserStatisticsReset } = await import("@/lib/user-statistics-reset/reset-queue"); + enqueueStarted = true; + const reset = await enqueueUserStatisticsReset(userId); + logger.info("Queued user statistics reset", { + userId, + resetId: reset.resetId, + status: reset.status, }); - // Invalidate auth cache outside transaction (Redis, not DB) - await invalidateCachedUser(userId).catch(() => {}); - - // 2. Clear Redis cache (cost keys + active sessions) - try { - const { clearUserCostCache } = await import("@/lib/redis/cost-cache-cleanup"); - const cacheResult = await clearUserCostCache({ - userId, - keyIds, - keyHashes, - includeActiveSessions: true, - }); - if (!cacheResult) { - logger.error("Reset user statistics committed DB changes without Redis cleanup", { - userId, - requiresRedisForFixed5h, - }); - return { - ok: false, - error: tError("USER_STATS_RESET_PARTIAL_FAILURE"), - errorCode: ERROR_CODES.USER_STATS_RESET_PARTIAL_FAILURE, - }; - } - - logger.info("Reset user statistics - Redis cache cleared", { - userId, - keyCount: keyIds.length, - ...cacheResult, - }); - if (cacheResult.cleanupFailed) { - return { - ok: false, - error: tError("USER_STATS_RESET_PARTIAL_FAILURE"), - errorCode: ERROR_CODES.USER_STATS_RESET_PARTIAL_FAILURE, - }; - } - } catch (error) { - logger.error("Failed to clear Redis cache during user statistics reset", { - userId, - error: error instanceof Error ? error.message : String(error), - }); - return { - ok: false, - error: tError("USER_STATS_RESET_PARTIAL_FAILURE"), - errorCode: ERROR_CODES.USER_STATS_RESET_PARTIAL_FAILURE, - }; - } - - logger.info("Reset all user statistics", { userId, keyCount: keyIds.length }); - revalidatePath("/dashboard/users"); - - return { ok: true }; + return { ok: true, data: reset }; } catch (error) { logger.error("Failed to reset all user statistics:", error); const tError = await getTranslations("errors"); return { ok: false, - error: tError("OPERATION_FAILED"), - errorCode: ERROR_CODES.OPERATION_FAILED, + error: tError(enqueueStarted ? "CONNECTION_FAILED" : "OPERATION_FAILED"), + errorCode: enqueueStarted ? ERROR_CODES.CONNECTION_FAILED : ERROR_CODES.OPERATION_FAILED, }; } } diff --git a/src/app/[locale]/dashboard/_components/user/edit-user-dialog.tsx b/src/app/[locale]/dashboard/_components/user/edit-user-dialog.tsx index 16a5f1442..083435d1f 100644 --- a/src/app/[locale]/dashboard/_components/user/edit-user-dialog.tsx +++ b/src/app/[locale]/dashboard/_components/user/edit-user-dialog.tsx @@ -4,7 +4,7 @@ import { useQueryClient } from "@tanstack/react-query"; import { Loader2, RotateCcw, Trash2, UserCog } from "lucide-react"; import { useRouter } from "next/navigation"; import { useLocale, useTranslations } from "next-intl"; -import { useMemo, useState, useTransition } from "react"; +import { useCallback, useEffect, useMemo, useState, useTransition } from "react"; import { toast } from "sonner"; import { z } from "zod"; import { @@ -29,12 +29,14 @@ import { } from "@/components/ui/dialog"; import { editUser, + getUserStatisticsReset, removeUser, resetUserAllStatistics, resetUserLimitsOnly, toggleUserEnabled, } from "@/lib/api-client/v1/actions/users"; import { useZodForm } from "@/lib/hooks/use-zod-form"; +import type { UserStatisticsResetRecord } from "@/lib/user-statistics-reset/types"; import { cn } from "@/lib/utils"; import { UpdateUserSchema } from "@/lib/validation/schemas"; import type { UserDisplay } from "@/types/user"; @@ -96,6 +98,7 @@ function EditUserDialogInner({ onOpenChange, user, onSuccess }: EditUserDialogPr const [isPending, startTransition] = useTransition(); const [isResettingAll, setIsResettingAll] = useState(false); const [resetAllDialogOpen, setResetAllDialogOpen] = useState(false); + const [statisticsReset, setStatisticsReset] = useState(null); const [isResetting5h, setIsResetting5h] = useState(false); const [reset5hDialogOpen, setReset5hDialogOpen] = useState(false); const [isResettingLimits, setIsResettingLimits] = useState(false); @@ -236,27 +239,75 @@ function EditUserDialogInner({ onOpenChange, user, onSuccess }: EditUserDialogPr router.refresh(); }; + const applyStatisticsResetStatus = useCallback( + (reset: UserStatisticsResetRecord): boolean => { + setStatisticsReset(reset); + if (reset.status === "completed") { + setIsResettingAll(false); + toast.success(t("editDialog.resetData.success")); + onSuccess?.(); + queryClient.invalidateQueries({ queryKey: ["users"] }); + router.refresh(); + return true; + } + if (reset.status === "failed") { + setIsResettingAll(false); + toast.error(t("editDialog.resetData.failed")); + return true; + } + return false; + }, + [onSuccess, queryClient, router, t] + ); + const handleResetAllStatistics = async () => { setIsResettingAll(true); try { const res = await resetUserAllStatistics(user.id); if (!res.ok) { + setIsResettingAll(false); toast.error(res.error || t("editDialog.resetData.error")); return; } - toast.success(t("editDialog.resetData.success")); + applyStatisticsResetStatus(res.data as UserStatisticsResetRecord); setResetAllDialogOpen(false); - - // Full page reload to ensure all cached data is refreshed - window.location.reload(); } catch (error) { + setIsResettingAll(false); console.error("[EditUserDialog] reset all statistics failed", error); toast.error(t("editDialog.resetData.error")); - } finally { - setIsResettingAll(false); } }; + useEffect(() => { + if (!statisticsReset || !["queued", "running"].includes(statisticsReset.status)) return; + let cancelled = false; + let timer: ReturnType | undefined; + + const poll = async () => { + const result = await getUserStatisticsReset(user.id, statisticsReset.resetId); + if (cancelled) return; + if (!result.ok) { + if (["CONNECTION_FAILED", "NETWORK_ERROR", "TIMEOUT"].includes(result.errorCode ?? "")) { + timer = setTimeout(poll, 2_000); + return; + } + setIsResettingAll(false); + toast.error(result.error || t("editDialog.resetData.error")); + return; + } + + const next = result.data as UserStatisticsResetRecord; + if (applyStatisticsResetStatus(next)) return; + timer = setTimeout(poll, 1_000); + }; + + timer = setTimeout(poll, 1_000); + return () => { + cancelled = true; + if (timer) clearTimeout(timer); + }; + }, [applyStatisticsResetStatus, statisticsReset, t, user.id]); + const handleResetLimitsOnly = async () => { setIsResettingLimits(true); try { @@ -489,13 +540,24 @@ function EditUserDialogInner({ onOpenChange, user, onSuccess }: EditUserDialogPr

{t("editDialog.resetData.description")}

+ {statisticsReset ? ( +

+ {t(`editDialog.resetData.${statisticsReset.status}`)} +

+ ) : null}
- diff --git a/src/app/api/v1/resources/users/handlers.ts b/src/app/api/v1/resources/users/handlers.ts index 9216c90a4..ef6052786 100644 --- a/src/app/api/v1/resources/users/handlers.ts +++ b/src/app/api/v1/resources/users/handlers.ts @@ -20,6 +20,7 @@ import { UserIdParamSchema, UserListQuerySchema, UserRenewSchema, + UserStatisticsResetParamsSchema, UsersBatchUpdateSchema, UsersUsageBatchSchema, UserUpdateSchema, @@ -225,7 +226,39 @@ export async function resetUserStatistics(c: Context): Promise { c.get("auth") ); if (!result.ok) return actionError(c, result); - return noContentResponse(); + const reset = result.data as { resetId: string }; + const location = `/api/v1/users/${params.id}/statistics-resets/${reset.resetId}`; + return jsonResponse(result.data, { status: 202, headers: { Location: location } }); +} + +export async function getUserStatisticsReset(c: Context): Promise { + const params = UserStatisticsResetParamsSchema.safeParse({ + id: c.req.param("id"), + resetId: c.req.param("resetId"), + }); + if (!params.success) return fromZodError(params.error, new URL(c.req.url).pathname); + + const { findUserStatisticsReset } = await import("@/lib/user-statistics-reset/reset-queue"); + let reset; + try { + reset = await findUserStatisticsReset(params.data.id, params.data.resetId); + } catch { + return createProblemResponse({ + status: 503, + instance: new URL(c.req.url).pathname, + errorCode: "dependency.unavailable", + detail: publicActionErrorDetail(503), + }); + } + if (!reset) { + return createProblemResponse({ + status: 404, + instance: new URL(c.req.url).pathname, + errorCode: "user.statistics_reset_not_found", + detail: "Statistics reset not found.", + }); + } + return jsonResponse(reset); } export async function getUserTags(c: Context): Promise { diff --git a/src/app/api/v1/resources/users/router.ts b/src/app/api/v1/resources/users/router.ts index d08cd15aa..98eac99eb 100644 --- a/src/app/api/v1/resources/users/router.ts +++ b/src/app/api/v1/resources/users/router.ts @@ -15,6 +15,8 @@ import { UserListQuerySchema, UserListResponseSchema, UserRenewSchema, + UserStatisticsResetParamsSchema, + UserStatisticsResetResponseSchema, UsersBatchUpdateSchema, UsersUsageBatchSchema, UserUpdateSchema, @@ -29,6 +31,7 @@ import { getUserAllLimitUsage, getUserKeyGroups, getUserLimitUsage, + getUserStatisticsReset, getUsersUsage, getUserTags, listCurrentUser, @@ -462,7 +465,41 @@ usersRouter.openapi( "x-required-access": "admin", security, request: { params: UserIdParamSchema }, - responses: { 204: { description: "User statistics reset." }, ...problemResponses }, + responses: { + 202: { + description: "User statistics reset queued.", + headers: { + Location: { + description: "Status resource for the queued statistics reset.", + schema: { type: "string" }, + }, + }, + content: { "application/json": { schema: UserStatisticsResetResponseSchema } }, + }, + ...problemResponses, + }, }), resetUserStatistics as never ); + +usersRouter.openapi( + createRoute({ + method: "get", + path: "/users/{id}/statistics-resets/{resetId}", + middleware: requireAuth("admin"), + tags: ["Users"], + summary: "Get user statistics reset status", + description: "Returns the durable status of an asynchronous statistics reset.", + "x-required-access": "admin", + security, + request: { params: UserStatisticsResetParamsSchema }, + responses: { + 200: { + description: "User statistics reset status.", + content: { "application/json": { schema: UserStatisticsResetResponseSchema } }, + }, + ...problemResponses, + }, + }), + getUserStatisticsReset as never +); diff --git a/src/lib/api-client/v1/actions/users.ts b/src/lib/api-client/v1/actions/users.ts index e4da0b95b..9b2a53f0d 100644 --- a/src/lib/api-client/v1/actions/users.ts +++ b/src/lib/api-client/v1/actions/users.ts @@ -164,7 +164,14 @@ export function resetUserLimitsOnly(userId: number) { } export function resetUserAllStatistics(userId: number) { - return toVoidActionResult(apiPost(`/api/v1/users/${userId}/statistics:reset`)); + return toActionResult(apiPost(`/api/v1/users/${userId}/statistics:reset`)); +} + +export async function getUserStatisticsReset(userId: number, resetId: string) { + const result = await toActionResult( + apiGet(`/api/v1/users/${userId}/statistics-resets/${encodeURIComponent(resetId)}`) + ); + return !result.ok && !result.errorCode ? { ...result, errorCode: "NETWORK_ERROR" } : result; } export function batchUpdateUsers(data: BatchUpdateUsersParams) { diff --git a/src/lib/api-client/v1/errors.ts b/src/lib/api-client/v1/errors.ts index 993b427b0..bb124488d 100644 --- a/src/lib/api-client/v1/errors.ts +++ b/src/lib/api-client/v1/errors.ts @@ -38,6 +38,7 @@ const API_ERROR_MESSAGE_KEYS: Record = { "auth.api_key_admin_disabled": "PERMISSION_DENIED", "auth.csrf_invalid": "PERMISSION_DENIED", "request.validation_failed": "INVALID_FORMAT", + "dependency.unavailable": "CONNECTION_FAILED", "resource.not_found": "NOT_FOUND", "provider.not_found": "NOT_FOUND", "provider.action_failed": "OPERATION_FAILED", diff --git a/src/lib/api-client/v1/openapi-types.gen.ts b/src/lib/api-client/v1/openapi-types.gen.ts index da3070baa..82c1038ef 100644 --- a/src/lib/api-client/v1/openapi-types.gen.ts +++ b/src/lib/api-client/v1/openapi-types.gen.ts @@ -2484,6 +2484,26 @@ export interface paths { patch?: never; trace?: never; }; + "/api/v1/users/{id}/statistics-resets/{resetId}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Get user statistics reset status + * @description Returns the durable status of an asynchronous statistics reset. + */ + get: operations["getUsersByIdStatisticsResetsByResetid"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/v1/users/{userId}/keys": { parameters: { query?: never; @@ -32789,12 +32809,293 @@ export interface operations { }; requestBody?: never; responses: { - /** @description User statistics reset. */ - 204: { + /** @description User statistics reset queued. */ + 202: { headers: { + /** @description Status resource for the queued statistics reset. */ + Location?: string; [name: string]: unknown; }; - content?: never; + content: { + "application/json": { + /** Format: uuid */ + resetId: string; + userId: number; + /** @enum {string} */ + status: "queued" | "running" | "completed" | "failed"; + /** Format: date-time */ + requestedAt: string; + /** Format: date-time */ + startedAt: string | null; + /** Format: date-time */ + completedAt: string | null; + deletedMessageRequests: number; + deletedUsageLedger: number; + errorCode: string | null; + }; + }; + }; + /** @description Invalid request. */ + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": { + /** @description Stable problem type URI or URN. */ + type: string; + /** @description Short problem title. */ + title: string; + /** @description HTTP status code. */ + status: number; + /** @description Human-readable error detail. */ + detail: string; + /** @description Request path that produced the problem. */ + instance: string; + /** @description Application error code for frontend i18n. */ + errorCode: string; + /** @description Optional i18n parameters. */ + errorParams?: { + [key: string]: unknown; + }; + /** @description Optional request trace identifier. */ + traceId?: string; + /** @description Validation failure details. */ + invalidParams?: { + /** @description Path to the invalid input field. */ + path: (string | number)[]; + /** @description Machine-readable validation error code. */ + code: string; + /** @description Validation error message. */ + message: string; + }[]; + }; + }; + }; + /** @description Authentication required. */ + 401: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": { + /** @description Stable problem type URI or URN. */ + type: string; + /** @description Short problem title. */ + title: string; + /** @description HTTP status code. */ + status: number; + /** @description Human-readable error detail. */ + detail: string; + /** @description Request path that produced the problem. */ + instance: string; + /** @description Application error code for frontend i18n. */ + errorCode: string; + /** @description Optional i18n parameters. */ + errorParams?: { + [key: string]: unknown; + }; + /** @description Optional request trace identifier. */ + traceId?: string; + /** @description Validation failure details. */ + invalidParams?: { + /** @description Path to the invalid input field. */ + path: (string | number)[]; + /** @description Machine-readable validation error code. */ + code: string; + /** @description Validation error message. */ + message: string; + }[]; + }; + }; + }; + /** @description Admin access required. */ + 403: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": { + /** @description Stable problem type URI or URN. */ + type: string; + /** @description Short problem title. */ + title: string; + /** @description HTTP status code. */ + status: number; + /** @description Human-readable error detail. */ + detail: string; + /** @description Request path that produced the problem. */ + instance: string; + /** @description Application error code for frontend i18n. */ + errorCode: string; + /** @description Optional i18n parameters. */ + errorParams?: { + [key: string]: unknown; + }; + /** @description Optional request trace identifier. */ + traceId?: string; + /** @description Validation failure details. */ + invalidParams?: { + /** @description Path to the invalid input field. */ + path: (string | number)[]; + /** @description Machine-readable validation error code. */ + code: string; + /** @description Validation error message. */ + message: string; + }[]; + }; + }; + }; + /** @description User not found. */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": { + /** @description Stable problem type URI or URN. */ + type: string; + /** @description Short problem title. */ + title: string; + /** @description HTTP status code. */ + status: number; + /** @description Human-readable error detail. */ + detail: string; + /** @description Request path that produced the problem. */ + instance: string; + /** @description Application error code for frontend i18n. */ + errorCode: string; + /** @description Optional i18n parameters. */ + errorParams?: { + [key: string]: unknown; + }; + /** @description Optional request trace identifier. */ + traceId?: string; + /** @description Validation failure details. */ + invalidParams?: { + /** @description Path to the invalid input field. */ + path: (string | number)[]; + /** @description Machine-readable validation error code. */ + code: string; + /** @description Validation error message. */ + message: string; + }[]; + }; + }; + }; + /** @description Internal server error. */ + 500: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": { + /** @description Stable problem type URI or URN. */ + type: string; + /** @description Short problem title. */ + title: string; + /** @description HTTP status code. */ + status: number; + /** @description Human-readable error detail. */ + detail: string; + /** @description Request path that produced the problem. */ + instance: string; + /** @description Application error code for frontend i18n. */ + errorCode: string; + /** @description Optional i18n parameters. */ + errorParams?: { + [key: string]: unknown; + }; + /** @description Optional request trace identifier. */ + traceId?: string; + /** @description Validation failure details. */ + invalidParams?: { + /** @description Path to the invalid input field. */ + path: (string | number)[]; + /** @description Machine-readable validation error code. */ + code: string; + /** @description Validation error message. */ + message: string; + }[]; + }; + }; + }; + /** @description Dependency unavailable. */ + 503: { + headers: { + [name: string]: unknown; + }; + content: { + "application/problem+json": { + /** @description Stable problem type URI or URN. */ + type: string; + /** @description Short problem title. */ + title: string; + /** @description HTTP status code. */ + status: number; + /** @description Human-readable error detail. */ + detail: string; + /** @description Request path that produced the problem. */ + instance: string; + /** @description Application error code for frontend i18n. */ + errorCode: string; + /** @description Optional i18n parameters. */ + errorParams?: { + [key: string]: unknown; + }; + /** @description Optional request trace identifier. */ + traceId?: string; + /** @description Validation failure details. */ + invalidParams?: { + /** @description Path to the invalid input field. */ + path: (string | number)[]; + /** @description Machine-readable validation error code. */ + code: string; + /** @description Validation error message. */ + message: string; + }[]; + }; + }; + }; + }; + }; + getUsersByIdStatisticsResetsByResetid: { + parameters: { + query?: never; + header?: never; + path: { + /** @description User id. */ + id: number; + /** @description Statistics reset id. */ + resetId: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description User statistics reset status. */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + /** Format: uuid */ + resetId: string; + userId: number; + /** @enum {string} */ + status: "queued" | "running" | "completed" | "failed"; + /** Format: date-time */ + requestedAt: string; + /** Format: date-time */ + startedAt: string | null; + /** Format: date-time */ + completedAt: string | null; + deletedMessageRequests: number; + deletedUsageLedger: number; + errorCode: string | null; + }; + }; }; /** @description Invalid request. */ 400: { diff --git a/src/lib/api/v1/schemas/users.ts b/src/lib/api/v1/schemas/users.ts index 6a22b9d35..8a88f6e0e 100644 --- a/src/lib/api/v1/schemas/users.ts +++ b/src/lib/api/v1/schemas/users.ts @@ -66,6 +66,22 @@ export const UserIdParamSchema = z.object({ id: z.coerce.number().int().positive().describe("User id."), }); +export const UserStatisticsResetParamsSchema = UserIdParamSchema.extend({ + resetId: z.string().uuid().describe("Statistics reset id."), +}); + +export const UserStatisticsResetResponseSchema = z.object({ + resetId: z.string().uuid(), + userId: z.number().int().positive(), + status: z.enum(["queued", "running", "completed", "failed"]), + requestedAt: z.string().datetime(), + startedAt: z.string().datetime().nullable(), + completedAt: z.string().datetime().nullable(), + deletedMessageRequests: z.number().int().nonnegative(), + deletedUsageLedger: z.number().int().nonnegative(), + errorCode: z.string().nullable(), +}); + export const UserListQuerySchema = z.object({ cursor: z.string().optional().describe("Cursor for admin batch listing."), limit: z.coerce.number().int().min(1).max(100).default(50).describe("Page size."), diff --git a/src/lib/redis/cost-cache-cleanup.ts b/src/lib/redis/cost-cache-cleanup.ts index c0e384557..e7a64bde9 100644 --- a/src/lib/redis/cost-cache-cleanup.ts +++ b/src/lib/redis/cost-cache-cleanup.ts @@ -9,6 +9,7 @@ export interface ClearUserCostCacheOptions { keyIds: number[]; keyHashes: string[]; includeActiveSessions?: boolean; + allowWhenRateLimitDisabled?: boolean; } export interface ClearUserCostCacheResult { @@ -52,29 +53,39 @@ export interface ClearUser5hCostCacheResult { export async function clearUserCostCache( options: ClearUserCostCacheOptions ): Promise { - const { userId, keyIds, keyHashes, includeActiveSessions = false } = options; - - const redis = getRedisClient(); + const { + userId, + keyIds, + keyHashes, + includeActiveSessions = false, + allowWhenRateLimitDisabled = false, + } = options; + + const redis = getRedisClient({ allowWhenRateLimitDisabled }); if (redis?.status !== "ready") { return null; } const startTime = Date.now(); + let scanErrorCount = 0; // Scan all cost patterns in parallel const scanResults = await Promise.all([ ...keyIds.map((keyId) => scanPattern(redis, `key:${keyId}:cost_*`).catch((err) => { + scanErrorCount += 1; logger.warn("Failed to scan key cost pattern", { keyId, error: err }); return []; }) ), scanPattern(redis, `user:${userId}:cost_*`).catch((err) => { + scanErrorCount += 1; logger.warn("Failed to scan user cost pattern", { userId, error: err }); return []; }), // Total cost cache keys (with optional resetAt suffix) scanPattern(redis, `total_cost:user:${userId}`).catch((err) => { + scanErrorCount += 1; logger.warn("Failed to scan total cost pattern", { userId, pattern: `total_cost:user:${userId}`, @@ -83,6 +94,7 @@ export async function clearUserCostCache( return []; }), scanPattern(redis, `total_cost:user:${userId}:*`).catch((err) => { + scanErrorCount += 1; logger.warn("Failed to scan total cost pattern", { userId, pattern: `total_cost:user:${userId}:*`, @@ -92,6 +104,7 @@ export async function clearUserCostCache( }), ...keyHashes.map((keyHash) => scanPattern(redis, `total_cost:key:${keyHash}`).catch((err) => { + scanErrorCount += 1; logger.warn("Failed to scan total cost key pattern", { keyHash, error: err instanceof Error ? err.message : String(err), @@ -101,6 +114,7 @@ export async function clearUserCostCache( ), ...keyHashes.map((keyHash) => scanPattern(redis, `total_cost:key:${keyHash}:*`).catch((err) => { + scanErrorCount += 1; logger.warn("Failed to scan total cost key pattern", { keyHash, error: err instanceof Error ? err.message : String(err), @@ -111,6 +125,7 @@ export async function clearUserCostCache( // Lease cache keys (budget slices cached by LeaseService) ...keyIds.map((keyId) => scanPattern(redis, `lease:key:${keyId}:*`).catch((err) => { + scanErrorCount += 1; logger.warn("Failed to scan lease key pattern", { keyId, error: err instanceof Error ? err.message : String(err), @@ -119,6 +134,7 @@ export async function clearUserCostCache( }) ), scanPattern(redis, `lease:user:${userId}:*`).catch((err) => { + scanErrorCount += 1; logger.warn("Failed to scan lease user pattern", { userId, error: err instanceof Error ? err.message : String(err), @@ -136,6 +152,8 @@ export async function clearUserCostCache( costKeysDeleted: 0, activeSessionsDeleted: 0, durationMs: Date.now() - startTime, + cleanupFailed: scanErrorCount > 0, + errorCount: scanErrorCount, }; } @@ -170,9 +188,10 @@ export async function clearUserCostCache( // Check for pipeline errors const errors = results?.filter(([err]) => err); - if (errors && errors.length > 0) { + const errorCount = scanErrorCount + (errors?.length ?? 0); + if (errorCount > 0) { logger.warn("Some Redis deletes failed during cost cache cleanup", { - errorCount: errors.length, + errorCount, userId, }); } @@ -181,8 +200,8 @@ export async function clearUserCostCache( costKeysDeleted: allCostKeys.length, activeSessionsDeleted, durationMs: Date.now() - startTime, - cleanupFailed: !!errors && errors.length > 0, - errorCount: errors?.length || 0, + cleanupFailed: errorCount > 0, + errorCount, }; } diff --git a/src/lib/user-statistics-reset/reset-queue.ts b/src/lib/user-statistics-reset/reset-queue.ts new file mode 100644 index 000000000..619e4303b --- /dev/null +++ b/src/lib/user-statistics-reset/reset-queue.ts @@ -0,0 +1,244 @@ +import "server-only"; + +import { randomUUID } from "node:crypto"; +import type { Job } from "bull"; +import Queue from "bull"; +import { logger } from "@/lib/logger"; +import { buildRedisQueueOptions } from "@/lib/redis/bull-queue-options"; +import { executeUserStatisticsReset, UserStatisticsResetError } from "./reset-service"; +import { + claimActiveUserStatisticsReset, + deleteUserStatisticsResetStatus, + getUserStatisticsResetStatus, + releaseActiveUserStatisticsReset, + setUserStatisticsResetStatus, +} from "./reset-status-store"; +import type { UserStatisticsResetJobData, UserStatisticsResetRecord } from "./types"; + +let resetQueue: Queue.Queue | null = null; +const RESET_JOB_NAME = "reset"; +const STALLED_FAILURE_REASON = "job stalled more than allowable limit"; + +function errorCode(error: unknown): string { + return error instanceof UserStatisticsResetError + ? error.code + : "USER_STATISTICS_RESET_OPERATION_FAILED"; +} + +function createQueuedRecord(input: UserStatisticsResetJobData): UserStatisticsResetRecord { + return { + ...input, + status: "queued", + startedAt: null, + completedAt: null, + deletedMessageRequests: 0, + deletedUsageLedger: 0, + errorCode: null, + }; +} + +function getResetQueue(): Queue.Queue { + if (resetQueue) return resetQueue; + const redisUrl = process.env.REDIS_URL; + if (!redisUrl) { + throw new Error("REDIS_URL environment variable is required for user statistics reset queue"); + } + + resetQueue = new Queue("user-statistics-reset", { + redis: buildRedisQueueOptions(redisUrl, "[UserStatisticsResetQueue]"), + defaultJobOptions: { + attempts: 5, + backoff: { type: "exponential", delay: 30_000 }, + removeOnComplete: 100, + removeOnFail: 100, + }, + }); + resetQueue.process(RESET_JOB_NAME, processUserStatisticsReset); + resetQueue.on("failed", async (job, error) => { + logger.error("[UserStatisticsResetQueue] job failed", { + resetId: job.data.resetId, + userId: job.data.userId, + attemptsMade: job.attemptsMade, + error: error.message, + }); + const attempts = job.opts.attempts ?? 1; + const isTerminal = job.attemptsMade >= attempts || error.message === STALLED_FAILURE_REASON; + if (isTerminal) { + try { + await recordFinalFailure(job.data, error); + } catch (statusError) { + logger.error("[UserStatisticsResetQueue] failed to persist terminal status", { + resetId: job.data.resetId, + userId: job.data.userId, + error: statusError instanceof Error ? statusError.message : String(statusError), + }); + } + } + }); + return resetQueue; +} + +async function recordFinalFailure( + jobData: UserStatisticsResetJobData, + error: unknown +): Promise { + const current = + (await getUserStatisticsResetStatus(jobData.resetId)) ?? createQueuedRecord(jobData); + await setUserStatisticsResetStatus({ + ...current, + status: "failed", + completedAt: new Date().toISOString(), + errorCode: errorCode(error), + }); + await releaseActiveUserStatisticsReset(jobData.userId, jobData.resetId); +} + +async function processUserStatisticsReset(job: Job) { + let current = + (await getUserStatisticsResetStatus(job.data.resetId)) ?? createQueuedRecord(job.data); + const startedAt = current.startedAt ?? new Date().toISOString(); + current = { + ...current, + status: "running", + startedAt, + errorCode: null, + }; + + try { + await setUserStatisticsResetStatus(current); + const deleted = await executeUserStatisticsReset(job.data); + const completed: UserStatisticsResetRecord = { + ...current, + deletedMessageRequests: current.deletedMessageRequests + deleted.deletedMessageRequests, + deletedUsageLedger: current.deletedUsageLedger + deleted.deletedUsageLedger, + status: "completed", + startedAt, + completedAt: new Date().toISOString(), + errorCode: null, + }; + await setUserStatisticsResetStatus(completed); + await releaseActiveUserStatisticsReset(job.data.userId, job.data.resetId); + return completed; + } catch (error) { + const attempts = job.opts.attempts ?? 1; + const isFinalAttempt = job.attemptsMade + 1 >= attempts; + const progress = + error instanceof UserStatisticsResetError + ? error.progress + : { deletedMessageRequests: 0, deletedUsageLedger: 0 }; + await setUserStatisticsResetStatus({ + ...current, + deletedMessageRequests: current.deletedMessageRequests + progress.deletedMessageRequests, + deletedUsageLedger: current.deletedUsageLedger + progress.deletedUsageLedger, + status: isFinalAttempt ? "failed" : "queued", + startedAt, + completedAt: isFinalAttempt ? new Date().toISOString() : null, + errorCode: isFinalAttempt ? errorCode(error) : null, + }); + if (isFinalAttempt) { + await releaseActiveUserStatisticsReset(job.data.userId, job.data.resetId); + } + throw error; + } +} + +export async function enqueueUserStatisticsReset( + userId: number +): Promise { + return enqueueUserStatisticsResetWithReconciliation(userId, true); +} + +async function enqueueUserStatisticsResetWithReconciliation( + userId: number, + allowReconciliation: boolean +): Promise { + const queue = getResetQueue(); + const jobData: UserStatisticsResetJobData = { + resetId: randomUUID(), + userId, + requestedAt: new Date().toISOString(), + }; + const queued = createQueuedRecord(jobData); + await setUserStatisticsResetStatus(queued); + + const claim = await claimActiveUserStatisticsReset(userId, jobData.resetId); + if (!claim.acquired) { + await deleteUserStatisticsResetStatus(jobData.resetId); + const existing = await getUserStatisticsResetStatus(claim.resetId); + const existingIsActive = + existing?.userId === userId && ["queued", "running"].includes(existing.status); + if (!existingIsActive) { + await releaseActiveUserStatisticsReset(userId, claim.resetId); + if (allowReconciliation) { + return enqueueUserStatisticsResetWithReconciliation(userId, false); + } + throw new Error("USER_STATISTICS_RESET_ACTIVE_STATUS_MISSING"); + } + + const existingJob = await queue.getJob(existing.resetId); + const existingJobState = existingJob ? await existingJob.getState() : null; + if (existingJobState === "failed" || existingJobState === "completed") { + await releaseActiveUserStatisticsReset(userId, existing.resetId); + if (allowReconciliation) { + return enqueueUserStatisticsResetWithReconciliation(userId, false); + } + throw new Error("USER_STATISTICS_RESET_ACTIVE_JOB_TERMINAL"); + } + if (!existingJob) { + await queue.add( + RESET_JOB_NAME, + { + resetId: existing.resetId, + userId: existing.userId, + requestedAt: existing.requestedAt, + }, + { jobId: existing.resetId } + ); + } + return existing; + } + + try { + await queue.add(RESET_JOB_NAME, jobData, { jobId: jobData.resetId }); + return queued; + } catch (error) { + await setUserStatisticsResetStatus({ + ...queued, + status: "failed", + completedAt: new Date().toISOString(), + errorCode: "USER_STATISTICS_RESET_QUEUE_FAILED", + }); + await releaseActiveUserStatisticsReset(userId, jobData.resetId); + throw error; + } +} + +export async function findUserStatisticsReset( + userId: number, + resetId: string +): Promise { + const record = await getUserStatisticsResetStatus(resetId); + return record?.userId === userId ? record : null; +} + +export function startUserStatisticsResetQueue(): boolean { + if (!process.env.REDIS_URL) { + logger.warn("[UserStatisticsResetQueue] disabled because REDIS_URL is not configured"); + return false; + } + try { + getResetQueue(); + return true; + } catch (error) { + logger.error("[UserStatisticsResetQueue] failed to start", { + error: error instanceof Error ? error.message : String(error), + }); + return false; + } +} + +export async function stopUserStatisticsResetQueue(): Promise { + if (!resetQueue) return; + await resetQueue.close(); + resetQueue = null; +} diff --git a/src/lib/user-statistics-reset/reset-service.ts b/src/lib/user-statistics-reset/reset-service.ts new file mode 100644 index 000000000..514f1c05e --- /dev/null +++ b/src/lib/user-statistics-reset/reset-service.ts @@ -0,0 +1,215 @@ +import "server-only"; + +import { and, eq, isNull, sql } from "drizzle-orm"; +import { db } from "@/drizzle/db"; +import { keys, users } from "@/drizzle/schema"; +import { clearUserCostCache } from "@/lib/redis/cost-cache-cleanup"; +import { invalidateCachedUser } from "@/lib/security/api-key-auth-cache"; + +const RESET_BATCH_SIZE = 1000; + +export class UserStatisticsResetError extends Error { + constructor( + readonly code: string, + readonly progress: { + deletedMessageRequests: number; + deletedUsageLedger: number; + } = { deletedMessageRequests: 0, deletedUsageLedger: 0 } + ) { + super(code); + this.name = "UserStatisticsResetError"; + } +} + +function affectedRows(result: unknown): number { + if (Array.isArray(result)) return result.length; + if (!result || typeof result !== "object") return 0; + const count = + (result as { count?: unknown; rowCount?: unknown }).count ?? + (result as { rowCount?: unknown }).rowCount; + return count === undefined ? 0 : Number(count); +} + +function firstRow(result: unknown): Record | undefined { + if (Array.isArray(result)) return result[0] as Record | undefined; + if (result && typeof result === "object" && Symbol.iterator in result) { + return Array.from(result as Iterable>)[0]; + } + return undefined; +} + +async function deleteMessageRequestBatch(userId: number, cutoff: Date): Promise { + return db.transaction(async (tx) => { + const result = await tx.execute(sql` + WITH doomed AS ( + SELECT id + FROM message_request + WHERE user_id = ${userId} + AND (created_at IS NULL OR created_at <= ${cutoff}) + LIMIT ${RESET_BATCH_SIZE} + FOR UPDATE SKIP LOCKED + ) + DELETE FROM message_request mr + USING doomed + WHERE mr.id = doomed.id + RETURNING 1 + `); + return affectedRows(result); + }); +} + +async function deleteUsageLedgerBatch(userId: number, cutoff: Date): Promise { + return db.transaction(async (tx) => { + const result = await tx.execute(sql` + WITH doomed AS ( + SELECT id + FROM usage_ledger + WHERE user_id = ${userId} + AND created_at <= ${cutoff} + LIMIT ${RESET_BATCH_SIZE} + FOR UPDATE SKIP LOCKED + ) + DELETE FROM usage_ledger ul + USING doomed + WHERE ul.id = doomed.id + RETURNING 1 + `); + return affectedRows(result); + }); +} + +async function hasRemainingRows( + table: "message_request" | "usage_ledger", + userId: number, + cutoff: Date +): Promise { + const tableName = + table === "message_request" ? sql.raw("message_request") : sql.raw("usage_ledger"); + const cutoffPredicate = + table === "message_request" + ? sql`(created_at IS NULL OR created_at <= ${cutoff})` + : sql`created_at <= ${cutoff}`; + const result = await db.execute(sql` + SELECT EXISTS ( + SELECT 1 FROM ${tableName} + WHERE user_id = ${userId} + AND ${cutoffPredicate} + ) AS "exists" + `); + return firstRow(result)?.exists === true; +} + +async function drainTable(input: { + table: "message_request" | "usage_ledger"; + userId: number; + cutoff: Date; +}): Promise { + let deleted = 0; + try { + while (true) { + const batchDeleted = + input.table === "message_request" + ? await deleteMessageRequestBatch(input.userId, input.cutoff) + : await deleteUsageLedgerBatch(input.userId, input.cutoff); + deleted += batchDeleted; + if (batchDeleted < RESET_BATCH_SIZE) break; + } + + if (!(await hasRemainingRows(input.table, input.userId, input.cutoff))) { + return deleted; + } + throw new UserStatisticsResetError( + "USER_STATISTICS_RESET_ROWS_LOCKED", + input.table === "message_request" + ? { deletedMessageRequests: deleted, deletedUsageLedger: 0 } + : { deletedMessageRequests: 0, deletedUsageLedger: deleted } + ); + } catch (error) { + if (error instanceof UserStatisticsResetError) throw error; + throw new UserStatisticsResetError( + "USER_STATISTICS_RESET_OPERATION_FAILED", + input.table === "message_request" + ? { deletedMessageRequests: deleted, deletedUsageLedger: 0 } + : { deletedMessageRequests: 0, deletedUsageLedger: deleted } + ); + } +} + +export async function executeUserStatisticsReset(input: { + userId: number; + requestedAt: string; +}): Promise<{ deletedMessageRequests: number; deletedUsageLedger: number }> { + const cutoff = new Date(input.requestedAt); + if (!Number.isFinite(cutoff.getTime())) { + throw new UserStatisticsResetError("USER_STATISTICS_RESET_INVALID_CUTOFF"); + } + + let deletedMessageRequests = 0; + let deletedUsageLedger = 0; + try { + deletedMessageRequests = await drainTable({ + table: "message_request", + userId: input.userId, + cutoff, + }); + deletedUsageLedger = await drainTable({ + table: "usage_ledger", + userId: input.userId, + cutoff, + }); + } catch (error) { + if (error instanceof UserStatisticsResetError) { + throw new UserStatisticsResetError(error.code, { + deletedMessageRequests: deletedMessageRequests + error.progress.deletedMessageRequests, + deletedUsageLedger: deletedUsageLedger + error.progress.deletedUsageLedger, + }); + } + if (deletedMessageRequests > 0 || deletedUsageLedger > 0) { + throw new UserStatisticsResetError("USER_STATISTICS_RESET_OPERATION_FAILED", { + deletedMessageRequests, + deletedUsageLedger, + }); + } + throw error; + } + + try { + const userKeys = await db + .select({ id: keys.id, key: keys.key }) + .from(keys) + .where(and(eq(keys.userId, input.userId), isNull(keys.deletedAt))); + + await db + .update(users) + .set({ + costResetAt: sql`CASE WHEN ${users.costResetAt} IS NULL OR ${users.costResetAt} <= ${cutoff} THEN NULL ELSE ${users.costResetAt} END`, + limit5hCostResetAt: sql`CASE WHEN ${users.limit5hCostResetAt} IS NULL OR ${users.limit5hCostResetAt} <= ${cutoff} THEN NULL ELSE ${users.limit5hCostResetAt} END`, + updatedAt: new Date(), + }) + .where(and(eq(users.id, input.userId), isNull(users.deletedAt))); + await invalidateCachedUser(input.userId); + + const cacheResult = await clearUserCostCache({ + userId: input.userId, + keyIds: userKeys.map((key) => key.id), + keyHashes: userKeys.map((key) => key.key), + includeActiveSessions: false, + allowWhenRateLimitDisabled: true, + }); + if (!cacheResult || cacheResult.cleanupFailed) { + throw new UserStatisticsResetError("USER_STATISTICS_RESET_CACHE_CLEANUP_FAILED"); + } + } catch (error) { + throw new UserStatisticsResetError( + error instanceof UserStatisticsResetError + ? error.code + : "USER_STATISTICS_RESET_OPERATION_FAILED", + { + deletedMessageRequests, + deletedUsageLedger, + } + ); + } + + return { deletedMessageRequests, deletedUsageLedger }; +} diff --git a/src/lib/user-statistics-reset/reset-status-store.ts b/src/lib/user-statistics-reset/reset-status-store.ts new file mode 100644 index 000000000..eb0ac3818 --- /dev/null +++ b/src/lib/user-statistics-reset/reset-status-store.ts @@ -0,0 +1,82 @@ +import "server-only"; + +import type Redis from "ioredis"; +import { getRedisClient } from "@/lib/redis/client"; +import { RedisKVStore } from "@/lib/redis/redis-kv-store"; +import type { UserStatisticsResetRecord } from "./types"; + +const RESET_STATUS_TTL_SECONDS = 7 * 24 * 60 * 60; +const ACTIVE_RESET_PREFIX = "cch:user-statistics-reset:active:"; +const RESET_STATUS_PREFIX = "cch:user-statistics-reset:status:"; +const statusStore = new RedisKVStore({ + prefix: RESET_STATUS_PREFIX, + defaultTtlSeconds: RESET_STATUS_TTL_SECONDS, +}); + +type ResetRedis = Pick & { + eval(...args: [script: string, numkeys: number, ...keysAndArgs: string[]]): Promise; +}; + +const LUA_COMPARE_DELETE = ` +if redis.call('GET', KEYS[1]) == ARGV[1] then + return redis.call('DEL', KEYS[1]) +end +return 0`; + +function getReadyRedis(): ResetRedis { + const redis = getRedisClient({ allowWhenRateLimitDisabled: true }) as ResetRedis | null; + if (redis?.status !== "ready") { + throw new Error("USER_STATISTICS_RESET_REDIS_UNAVAILABLE"); + } + return redis; +} + +export async function setUserStatisticsResetStatus( + record: UserStatisticsResetRecord +): Promise { + if (!(await statusStore.set(record.resetId, record))) { + throw new Error("USER_STATISTICS_RESET_STATUS_WRITE_FAILED"); + } +} + +export async function getUserStatisticsResetStatus( + resetId: string +): Promise { + const raw = await getReadyRedis().get(`${RESET_STATUS_PREFIX}${resetId}`); + if (!raw) return null; + try { + return JSON.parse(raw) as UserStatisticsResetRecord; + } catch { + throw new Error("USER_STATISTICS_RESET_STATUS_INVALID"); + } +} + +export async function deleteUserStatisticsResetStatus(resetId: string): Promise { + await getReadyRedis().del(`${RESET_STATUS_PREFIX}${resetId}`); +} + +export async function claimActiveUserStatisticsReset( + userId: number, + resetId: string +): Promise<{ acquired: boolean; resetId: string }> { + const redis = getReadyRedis(); + const key = `${ACTIVE_RESET_PREFIX}${userId}`; + const result = await redis.set(key, resetId, "EX", RESET_STATUS_TTL_SECONDS, "NX"); + if (result === "OK") { + return { acquired: true, resetId }; + } + + const existing = await redis.get(key); + if (!existing) { + throw new Error("USER_STATISTICS_RESET_ACTIVE_CLAIM_FAILED"); + } + return { acquired: false, resetId: existing }; +} + +export async function releaseActiveUserStatisticsReset( + userId: number, + resetId: string +): Promise { + const redis = getReadyRedis(); + await redis.eval(LUA_COMPARE_DELETE, 1, `${ACTIVE_RESET_PREFIX}${userId}`, resetId); +} diff --git a/src/lib/user-statistics-reset/types.ts b/src/lib/user-statistics-reset/types.ts new file mode 100644 index 000000000..a0ce19e7e --- /dev/null +++ b/src/lib/user-statistics-reset/types.ts @@ -0,0 +1,19 @@ +export type UserStatisticsResetStatus = "queued" | "running" | "completed" | "failed"; + +export interface UserStatisticsResetRecord { + resetId: string; + userId: number; + status: UserStatisticsResetStatus; + requestedAt: string; + startedAt: string | null; + completedAt: string | null; + deletedMessageRequests: number; + deletedUsageLedger: number; + errorCode: string | null; +} + +export interface UserStatisticsResetJobData { + resetId: string; + userId: number; + requestedAt: string; +} diff --git a/tests/api/v1/users/users.test.ts b/tests/api/v1/users/users.test.ts index 4b80db618..792300c7a 100644 --- a/tests/api/v1/users/users.test.ts +++ b/tests/api/v1/users/users.test.ts @@ -17,6 +17,7 @@ const getUserLimitUsageMock = vi.hoisted(() => vi.fn()); const getUserAllLimitUsageMock = vi.hoisted(() => vi.fn()); const resetUserLimitsOnlyMock = vi.hoisted(() => vi.fn()); const resetUserAllStatisticsMock = vi.hoisted(() => vi.fn()); +const findUserStatisticsResetMock = vi.hoisted(() => vi.fn()); const getAllUserTagsMock = vi.hoisted(() => vi.fn()); const getAllUserKeyGroupsMock = vi.hoisted(() => vi.fn()); const searchUsersForFilterMock = vi.hoisted(() => vi.fn()); @@ -51,6 +52,10 @@ vi.mock("@/actions/users", () => ({ batchUpdateUsers: batchUpdateUsersMock, })); +vi.mock("@/lib/user-statistics-reset/reset-queue", () => ({ + findUserStatisticsReset: findUserStatisticsResetMock, +})); + const { callV1Route } = await import("../test-utils"); const adminSession = { @@ -115,7 +120,31 @@ describe("v1 users endpoints", () => { data: { limitDaily: { usage: 1, limit: 10 } }, }); resetUserLimitsOnlyMock.mockResolvedValue({ ok: true }); - resetUserAllStatisticsMock.mockResolvedValue({ ok: true }); + resetUserAllStatisticsMock.mockResolvedValue({ + ok: true, + data: { + resetId: "00000000-0000-4000-8000-000000000001", + userId: 1, + status: "queued", + requestedAt: "2026-08-02T12:00:00.000Z", + startedAt: null, + completedAt: null, + deletedMessageRequests: 0, + deletedUsageLedger: 0, + errorCode: null, + }, + }); + findUserStatisticsResetMock.mockResolvedValue({ + resetId: "00000000-0000-4000-8000-000000000001", + userId: 1, + status: "running", + requestedAt: "2026-08-02T12:00:00.000Z", + startedAt: "2026-08-02T12:00:01.000Z", + completedAt: null, + deletedMessageRequests: 1000, + deletedUsageLedger: 0, + errorCode: null, + }); getAllUserTagsMock.mockResolvedValue({ ok: true, data: ["team-a"] }); getAllUserKeyGroupsMock.mockResolvedValue({ ok: true, data: ["default"] }); searchUsersForFilterMock.mockResolvedValue({ ok: true, data: [{ id: 1, name: "user-1" }] }); @@ -346,7 +375,19 @@ describe("v1 users endpoints", () => { pathname: "/api/v1/users/1/statistics:reset", headers, }); - expect(resetStats.response.status).toBe(204); + expect(resetStats.response.status).toBe(202); + expect(resetStats.response.headers.get("location")).toBe( + "/api/v1/users/1/statistics-resets/00000000-0000-4000-8000-000000000001" + ); + expect(resetStats.json).toMatchObject({ status: "queued" }); + + const resetStatus = await callV1Route({ + method: "GET", + pathname: "/api/v1/users/1/statistics-resets/00000000-0000-4000-8000-000000000001", + headers, + }); + expect(resetStatus.response.status).toBe(200); + expect(resetStatus.json).toMatchObject({ status: "running", deletedMessageRequests: 1000 }); }); test("maps structured authorization action errors to HTTP status codes", async () => { @@ -500,6 +541,7 @@ describe("v1 users endpoints", () => { expect(doc.paths).toHaveProperty("/api/v1/users/{id}/limit-usage:all"); expect(doc.paths).toHaveProperty("/api/v1/users/{id}/limits:reset"); expect(doc.paths).toHaveProperty("/api/v1/users/{id}/statistics:reset"); + expect(doc.paths).toHaveProperty("/api/v1/users/{id}/statistics-resets/{resetId}"); expect(doc.paths).toHaveProperty("/api/v1/users:batchUpdate"); expect(doc.paths).toHaveProperty("/api/v1/users:usageBatch"); expect(doc.paths).toHaveProperty("/api/v1/users:filter-search"); @@ -509,5 +551,9 @@ describe("v1 users endpoints", () => { expect(JSON.stringify(userDetail.get?.responses?.["200"])).toContain("createdAt"); expect(userDetail.get?.responses).toHaveProperty("500"); expect(userDetail.get?.responses).toHaveProperty("503"); + const resetStatistics = doc.paths["/api/v1/users/{id}/statistics:reset"] as { + post?: { responses?: Record }> }; + }; + expect(resetStatistics.post?.responses?.["202"]?.headers).toHaveProperty("Location"); }); }); diff --git a/tests/unit/actions/users-reset-5h-only.test.ts b/tests/unit/actions/users-reset-5h-only.test.ts index e167d7bdf..6ab46a5e5 100644 --- a/tests/unit/actions/users-reset-5h-only.test.ts +++ b/tests/unit/actions/users-reset-5h-only.test.ts @@ -66,6 +66,11 @@ vi.mock("@/lib/security/api-key-auth-cache", () => ({ invalidateCachedUser: invalidateCachedUserMock, })); +const enqueueUserStatisticsResetMock = vi.fn(); +vi.mock("@/lib/user-statistics-reset/reset-queue", () => ({ + enqueueUserStatisticsReset: enqueueUserStatisticsResetMock, +})); + const txDeleteWhereMock = vi.fn(); const txDeleteMock = vi.fn(() => ({ where: txDeleteWhereMock })); const txUpdateWhereMock = vi.fn(); @@ -286,6 +291,17 @@ describe("full reset compatibility with user 5h marker", () => { updateUserCostResetMarkersMock.mockResolvedValue(true); txUpdateWhereMock.mockResolvedValue([{ id: 123 }]); txDeleteWhereMock.mockResolvedValue([]); + enqueueUserStatisticsResetMock.mockResolvedValue({ + resetId: "00000000-0000-4000-8000-000000000001", + userId: 123, + status: "queued", + requestedAt: "2026-08-02T12:00:00.000Z", + startedAt: null, + completedAt: null, + deletedMessageRequests: 0, + deletedUsageLedger: 0, + errorCode: null, + }); }); test("full reset still resets all amount windows and advances 5h marker", async () => { @@ -301,18 +317,14 @@ describe("full reset compatibility with user 5h marker", () => { expect(clearUserCostCacheMock).toHaveBeenCalled(); }); - test("full statistics reset does not leave stale 5h marker", async () => { + test("full statistics reset queues marker cleanup in the background worker", async () => { const { resetUserAllStatistics } = await import("@/actions/users"); const result = await resetUserAllStatistics(123); expect(result.ok).toBe(true); - expect(txUpdateSetMock).toHaveBeenCalledWith( - expect.objectContaining({ - costResetAt: null, - limit5hCostResetAt: null, - }) - ); - expect(invalidateCachedUserMock).toHaveBeenCalledWith(123); + expect(enqueueUserStatisticsResetMock).toHaveBeenCalledWith(123); + expect(txUpdateSetMock).not.toHaveBeenCalled(); + expect(invalidateCachedUserMock).not.toHaveBeenCalled(); }); test("full statistics reset fails when fixed 5h state exists but Redis is unavailable", async () => { @@ -329,7 +341,7 @@ describe("full reset compatibility with user 5h marker", () => { const result = await resetUserAllStatistics(123); expect(result.ok).toBe(false); - expect(result.errorCode).toBe(ERROR_CODES.OPERATION_FAILED); + expect(result.errorCode).toBe(ERROR_CODES.CONNECTION_FAILED); expect(txUpdateSetMock).not.toHaveBeenCalled(); }); @@ -354,7 +366,7 @@ describe("full reset compatibility with user 5h marker", () => { const result = await resetUserAllStatistics(123); expect(result.ok).toBe(false); - expect(result.errorCode).toBe(ERROR_CODES.OPERATION_FAILED); + expect(result.errorCode).toBe(ERROR_CODES.CONNECTION_FAILED); expect(txUpdateSetMock).not.toHaveBeenCalled(); }); }); diff --git a/tests/unit/actions/users-reset-all-statistics.test.ts b/tests/unit/actions/users-reset-all-statistics.test.ts index a6280bf0a..50a6930ae 100644 --- a/tests/unit/actions/users-reset-all-statistics.test.ts +++ b/tests/unit/actions/users-reset-all-statistics.test.ts @@ -1,266 +1,109 @@ import { beforeEach, describe, expect, test, vi } from "vitest"; import { ERROR_CODES } from "@/lib/utils/error-messages"; -// Mock getSession -const getSessionMock = vi.fn(); -vi.mock("@/lib/auth", () => ({ - getSession: getSessionMock, +const mocks = vi.hoisted(() => ({ + getSession: vi.fn(), + findUserById: vi.fn(), + findKeyList: vi.fn(), + getRedisClient: vi.fn(), + enqueue: vi.fn(), })); -// Mock next-intl -const getTranslationsMock = vi.fn(async () => (key: string) => key); +vi.mock("@/lib/auth", () => ({ getSession: mocks.getSession })); vi.mock("next-intl/server", () => ({ - getTranslations: getTranslationsMock, + getTranslations: vi.fn(async () => (key: string) => key), getLocale: vi.fn(async () => "en"), })); - -// Mock next/cache -const revalidatePathMock = vi.fn(); -vi.mock("next/cache", () => ({ - revalidatePath: revalidatePathMock, -})); - -// Mock repository/user -const findUserByIdMock = vi.fn(); -const resetUserCostResetAtMock = vi.fn(); -vi.mock("@/repository/user", async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - findUserById: findUserByIdMock, - resetUserCostResetAt: resetUserCostResetAtMock, - }; -}); - -// Mock repository/key -const findKeyListMock = vi.fn(); -vi.mock("@/repository/key", async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - findKeyList: findKeyListMock, - }; -}); - -// Mock drizzle db -const txDeleteWhereMock = vi.fn(); -const txDeleteMock = vi.fn(() => ({ where: txDeleteWhereMock })); -const txUpdateSetMock = vi.fn(() => ({ where: vi.fn().mockResolvedValue(undefined) })); -const txUpdateMock = vi.fn(() => ({ set: txUpdateSetMock })); -const txMock = { - delete: txDeleteMock, - update: txUpdateMock, -}; -const dbTransactionMock = vi.fn(async (fn: (tx: typeof txMock) => Promise) => { - await fn(txMock); -}); -vi.mock("@/drizzle/db", () => ({ - db: { - transaction: dbTransactionMock, - }, +vi.mock("next/cache", () => ({ revalidatePath: vi.fn() })); +vi.mock("@/repository/user", async (importOriginal) => ({ + ...(await importOriginal()), + findUserById: mocks.findUserById, })); - -// Mock logger -const loggerMock = { - info: vi.fn(), - warn: vi.fn(), - error: vi.fn(), -}; -vi.mock("@/lib/logger", () => ({ - logger: loggerMock, +vi.mock("@/repository/key", async (importOriginal) => ({ + ...(await importOriginal()), + findKeyList: mocks.findKeyList, })); - -// Mock invalidateCachedUser (called directly after transaction) -const invalidateCachedUserMock = vi.fn(); -vi.mock("@/lib/security/api-key-auth-cache", () => ({ - invalidateCachedUser: invalidateCachedUserMock, +vi.mock("@/lib/redis", () => ({ getRedisClient: mocks.getRedisClient })); +vi.mock("@/lib/user-statistics-reset/reset-queue", () => ({ + enqueueUserStatisticsReset: mocks.enqueue, })); -// Mock Redis -const redisPipelineMock = { - del: vi.fn().mockReturnThis(), - exec: vi.fn(), +const queuedReset = { + resetId: "00000000-0000-4000-8000-000000000001", + userId: 123, + status: "queued" as const, + requestedAt: "2026-08-02T12:00:00.000Z", + startedAt: null, + completedAt: null, + deletedMessageRequests: 0, + deletedUsageLedger: 0, + errorCode: null, }; -const redisMock = { - status: "ready", - pipeline: vi.fn(() => redisPipelineMock), -}; -const getRedisClientMock = vi.fn(() => redisMock); -vi.mock("@/lib/redis", () => ({ - getRedisClient: getRedisClientMock, -})); - -// Mock scanPattern -const scanPatternMock = vi.fn(); -vi.mock("@/lib/redis/scan-helper", () => ({ - scanPattern: scanPatternMock, -})); describe("resetUserAllStatistics", () => { beforeEach(() => { vi.clearAllMocks(); - // Reset redis mock to ready state - redisMock.status = "ready"; - redisPipelineMock.exec.mockResolvedValue([]); - // DB delete returns resolved promise - txDeleteWhereMock.mockResolvedValue(undefined); - resetUserCostResetAtMock.mockResolvedValue(true); - invalidateCachedUserMock.mockResolvedValue(undefined); + mocks.getSession.mockResolvedValue({ user: { id: 1, role: "admin" } }); + mocks.findUserById.mockResolvedValue({ + id: 123, + limit5hUsd: null, + limit5hResetMode: "rolling", + }); + mocks.findKeyList.mockResolvedValue([]); + mocks.getRedisClient.mockReturnValue({ status: "ready" }); + mocks.enqueue.mockResolvedValue(queuedReset); }); - test("should return PERMISSION_DENIED for non-admin user", async () => { - getSessionMock.mockResolvedValue({ user: { id: 1, role: "user" } }); - + test("returns PERMISSION_DENIED for non-admin users", async () => { + mocks.getSession.mockResolvedValue({ user: { id: 1, role: "user" } }); const { resetUserAllStatistics } = await import("@/actions/users"); - const result = await resetUserAllStatistics(123); - expect(result.ok).toBe(false); - expect(result.errorCode).toBe(ERROR_CODES.PERMISSION_DENIED); - expect(findUserByIdMock).not.toHaveBeenCalled(); - }); - - test("should return PERMISSION_DENIED when no session", async () => { - getSessionMock.mockResolvedValue(null); - - const { resetUserAllStatistics } = await import("@/actions/users"); const result = await resetUserAllStatistics(123); - expect(result.ok).toBe(false); - expect(result.errorCode).toBe(ERROR_CODES.PERMISSION_DENIED); + expect(result).toMatchObject({ ok: false, errorCode: ERROR_CODES.PERMISSION_DENIED }); + expect(mocks.enqueue).not.toHaveBeenCalled(); }); - test("should return NOT_FOUND for non-existent user", async () => { - getSessionMock.mockResolvedValue({ user: { id: 1, role: "admin" } }); - findUserByIdMock.mockResolvedValue(null); - + test("returns NOT_FOUND for a missing user", async () => { + mocks.findUserById.mockResolvedValue(null); const { resetUserAllStatistics } = await import("@/actions/users"); - const result = await resetUserAllStatistics(999); - expect(result.ok).toBe(false); - expect(result.errorCode).toBe(ERROR_CODES.NOT_FOUND); - expect(dbTransactionMock).not.toHaveBeenCalled(); - }); - - test("should successfully reset all user statistics", async () => { - getSessionMock.mockResolvedValue({ user: { id: 1, role: "admin" } }); - findUserByIdMock.mockResolvedValue({ id: 123, name: "Test User" }); - findKeyListMock.mockResolvedValue([{ id: 1 }, { id: 2 }]); - scanPatternMock.mockResolvedValue(["key:1:cost_daily", "key:2:cost_weekly"]); - redisPipelineMock.exec.mockResolvedValue([]); - - const { resetUserAllStatistics } = await import("@/actions/users"); const result = await resetUserAllStatistics(123); - expect(result.ok).toBe(true); - // DB transaction called (delete + update wrapped in transaction) - expect(dbTransactionMock).toHaveBeenCalled(); - expect(txDeleteMock).toHaveBeenCalled(); - expect(txDeleteWhereMock).toHaveBeenCalled(); - // Redis operations - expect(redisMock.pipeline).toHaveBeenCalled(); - expect(redisPipelineMock.del).toHaveBeenCalled(); - expect(redisPipelineMock.exec).toHaveBeenCalled(); - // Revalidate path - expect(revalidatePathMock).toHaveBeenCalledWith("/dashboard/users"); - // Logging - expect(loggerMock.info).toHaveBeenCalled(); + expect(result).toMatchObject({ ok: false, errorCode: ERROR_CODES.NOT_FOUND }); }); - test("should return partial failure when Redis is not ready after DB reset", async () => { - getSessionMock.mockResolvedValue({ user: { id: 1, role: "admin" } }); - findUserByIdMock.mockResolvedValue({ id: 123, name: "Test User" }); - findKeyListMock.mockResolvedValue([{ id: 1 }]); - redisMock.status = "connecting"; - + test("keeps the fixed 5h Redis availability guard before enqueue", async () => { + mocks.findUserById.mockResolvedValue({ + id: 123, + limit5hUsd: 10, + limit5hResetMode: "fixed", + }); + mocks.getRedisClient.mockReturnValue(null); const { resetUserAllStatistics } = await import("@/actions/users"); - const result = await resetUserAllStatistics(123); - expect(result.ok).toBe(false); - expect(result.errorCode).toBe(ERROR_CODES.USER_STATS_RESET_PARTIAL_FAILURE); - expect(dbTransactionMock).toHaveBeenCalled(); - expect(redisMock.pipeline).not.toHaveBeenCalled(); - }); - - test("should return partial failure when Redis has partial failures", async () => { - getSessionMock.mockResolvedValue({ user: { id: 1, role: "admin" } }); - findUserByIdMock.mockResolvedValue({ id: 123, name: "Test User" }); - findKeyListMock.mockResolvedValue([{ id: 1 }]); - scanPatternMock.mockResolvedValue(["key:1:cost_daily"]); - // Simulate partial failure - some commands return errors - redisPipelineMock.exec.mockResolvedValue([ - [null, 1], // success - [new Error("Connection reset"), null], // failure - ]); - - const { resetUserAllStatistics } = await import("@/actions/users"); const result = await resetUserAllStatistics(123); - expect(result.ok).toBe(false); - expect(result.errorCode).toBe(ERROR_CODES.USER_STATS_RESET_PARTIAL_FAILURE); - expect(loggerMock.warn).toHaveBeenCalledWith( - "Some Redis deletes failed during cost cache cleanup", - expect.objectContaining({ errorCount: 1, userId: 123 }) - ); + expect(result).toMatchObject({ ok: false, errorCode: ERROR_CODES.CONNECTION_FAILED }); + expect(mocks.getRedisClient).toHaveBeenCalledWith({ allowWhenRateLimitDisabled: true }); + expect(mocks.enqueue).not.toHaveBeenCalled(); }); - test("should succeed with warning when scanPattern fails", async () => { - getSessionMock.mockResolvedValue({ user: { id: 1, role: "admin" } }); - findUserByIdMock.mockResolvedValue({ id: 123, name: "Test User" }); - findKeyListMock.mockResolvedValue([{ id: 1 }]); - // scanPattern fails but is caught by .catch() in Promise.all - scanPatternMock.mockRejectedValue(new Error("Redis connection lost")); - redisPipelineMock.exec.mockResolvedValue([]); - + test("queues the reset and returns its durable status", async () => { const { resetUserAllStatistics } = await import("@/actions/users"); - const result = await resetUserAllStatistics(123); - - // Should still succeed - error is caught inside Promise.all - expect(result.ok).toBe(true); - expect(loggerMock.warn).toHaveBeenCalled(); - }); - test("should return partial failure when pipeline.exec throws", async () => { - getSessionMock.mockResolvedValue({ user: { id: 1, role: "admin" } }); - findUserByIdMock.mockResolvedValue({ id: 123, name: "Test User" }); - findKeyListMock.mockResolvedValue([{ id: 1 }]); - scanPatternMock.mockResolvedValue(["key:1:cost_daily"]); - // pipeline.exec throws - caught inside clearUserCostCache (never-throws contract) - redisPipelineMock.exec.mockRejectedValue(new Error("Pipeline failed")); - - const { resetUserAllStatistics } = await import("@/actions/users"); const result = await resetUserAllStatistics(123); - expect(result.ok).toBe(false); - expect(result.errorCode).toBe(ERROR_CODES.USER_STATS_RESET_PARTIAL_FAILURE); - expect(loggerMock.warn).toHaveBeenCalledWith( - "Redis pipeline.exec() failed during cost cache cleanup", - expect.objectContaining({ userId: 123 }) - ); + expect(result).toEqual({ ok: true, data: queuedReset }); + expect(mocks.enqueue).toHaveBeenCalledWith(123); }); - test("should return OPERATION_FAILED on unexpected error", async () => { - getSessionMock.mockResolvedValue({ user: { id: 1, role: "admin" } }); - findUserByIdMock.mockRejectedValue(new Error("Database connection failed")); - + test("maps queue failures to a retryable dependency error", async () => { + mocks.enqueue.mockRejectedValue(new Error("redis unavailable")); const { resetUserAllStatistics } = await import("@/actions/users"); - const result = await resetUserAllStatistics(123); - - expect(result.ok).toBe(false); - expect(result.errorCode).toBe(ERROR_CODES.OPERATION_FAILED); - expect(loggerMock.error).toHaveBeenCalled(); - }); - test("should handle user with no keys", async () => { - getSessionMock.mockResolvedValue({ user: { id: 1, role: "admin" } }); - findUserByIdMock.mockResolvedValue({ id: 123, name: "Test User" }); - findKeyListMock.mockResolvedValue([]); // No keys - scanPatternMock.mockResolvedValue([]); - redisPipelineMock.exec.mockResolvedValue([]); - - const { resetUserAllStatistics } = await import("@/actions/users"); const result = await resetUserAllStatistics(123); - expect(result.ok).toBe(true); - expect(dbTransactionMock).toHaveBeenCalled(); + expect(result).toMatchObject({ ok: false, errorCode: ERROR_CODES.CONNECTION_FAILED }); }); }); diff --git a/tests/unit/api/v1/api-client-actions.test.ts b/tests/unit/api/v1/api-client-actions.test.ts index 4bad78fa2..b48f1de8e 100644 --- a/tests/unit/api/v1/api-client-actions.test.ts +++ b/tests/unit/api/v1/api-client-actions.test.ts @@ -68,6 +68,27 @@ describe("v1 action compatibility client", () => { ); }); + test("passes an AbortSignal to the all-sessions request", async () => { + getMock.mockResolvedValue({ active: [], inactive: [] }); + const controller = new AbortController(); + + await activeSessions.getAllSessions(2, 3, 20, { signal: controller.signal }); + + expect(getMock).toHaveBeenCalledWith( + "/api/v1/sessions?state=all&activePage=2&inactivePage=3&pageSize=20", + { signal: controller.signal } + ); + }); + + test("marks statistics reset polling transport failures as retryable network errors", async () => { + getMock.mockRejectedValue(new TypeError("Failed to fetch")); + + await expect(users.getUserStatisticsReset(42, "reset-id")).resolves.toMatchObject({ + ok: false, + errorCode: "NETWORK_ERROR", + }); + }); + test("preserves the physical request locator for every Session payload endpoint", async () => { getMock.mockResolvedValue({ exists: true, response: "ok" }); diff --git a/tests/unit/lib/redis/cost-cache-cleanup.test.ts b/tests/unit/lib/redis/cost-cache-cleanup.test.ts index 3fef0f4c7..b9e22a524 100644 --- a/tests/unit/lib/redis/cost-cache-cleanup.test.ts +++ b/tests/unit/lib/redis/cost-cache-cleanup.test.ts @@ -159,6 +159,22 @@ describe("clearUserCostCache", () => { expect(result).toBeNull(); }); + test("uses maintenance Redis and reports scan failures", async () => { + scanPatternMock.mockRejectedValue(new Error("scan failed")); + + const { clearUserCostCache } = await import("@/lib/redis/cost-cache-cleanup"); + const result = await clearUserCostCache({ + userId: 10, + keyIds: [], + keyHashes: [], + allowWhenRateLimitDisabled: true, + }); + + expect(getRedisClientMock).toHaveBeenCalledWith({ allowWhenRateLimitDisabled: true }); + expect(result).toMatchObject({ cleanupFailed: true, errorCount: 4 }); + expect(redisMock.pipeline).not.toHaveBeenCalled(); + }); + test("includeActiveSessions=true adds session key DELs", async () => { scanPatternMock.mockResolvedValue([]); diff --git a/tests/unit/lib/user-statistics-reset-queue.test.ts b/tests/unit/lib/user-statistics-reset-queue.test.ts new file mode 100644 index 000000000..985ccef4c --- /dev/null +++ b/tests/unit/lib/user-statistics-reset-queue.test.ts @@ -0,0 +1,279 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const boundary = vi.hoisted(() => ({ + processHandler: null as null | ((job: any) => Promise), + failedHandler: null as null | ((job: any, error: Error) => Promise), + processName: null as string | null, + queueOptions: null as any, + add: vi.fn(), + getJob: vi.fn(), + close: vi.fn(), + claim: vi.fn(), + getStatus: vi.fn(), + setStatus: vi.fn(), + deleteStatus: vi.fn(), + release: vi.fn(), + execute: vi.fn(), +})); + +vi.mock("bull", () => ({ + default: class MockQueue { + constructor(_name: string, options: unknown) { + boundary.queueOptions = options; + } + process(name: string, handler: (job: any) => Promise) { + boundary.processName = name; + boundary.processHandler = handler; + } + on(event: string, handler: (job: any, error: Error) => Promise) { + if (event === "failed") boundary.failedHandler = handler; + } + add = boundary.add; + getJob = boundary.getJob; + close = boundary.close; + }, +})); +vi.mock("@/lib/redis/bull-queue-options", () => ({ + buildRedisQueueOptions: () => ({ host: "redis" }), +})); +vi.mock("@/lib/user-statistics-reset/reset-status-store", () => ({ + claimActiveUserStatisticsReset: boundary.claim, + getUserStatisticsResetStatus: boundary.getStatus, + setUserStatisticsResetStatus: boundary.setStatus, + deleteUserStatisticsResetStatus: boundary.deleteStatus, + releaseActiveUserStatisticsReset: boundary.release, +})); +vi.mock("@/lib/user-statistics-reset/reset-service", () => ({ + executeUserStatisticsReset: boundary.execute, + UserStatisticsResetError: class UserStatisticsResetError extends Error { + constructor(readonly code: string) { + super(code); + } + }, +})); + +import { + enqueueUserStatisticsReset, + startUserStatisticsResetQueue, + stopUserStatisticsResetQueue, +} from "@/lib/user-statistics-reset/reset-queue"; + +const existing = { + resetId: "00000000-0000-4000-8000-000000000002", + userId: 42, + status: "running" as const, + requestedAt: "2026-08-02T12:00:00.000Z", + startedAt: "2026-08-02T12:00:01.000Z", + completedAt: null, + deletedMessageRequests: 1000, + deletedUsageLedger: 0, + errorCode: null, +}; + +describe("user statistics reset queue", () => { + beforeEach(async () => { + await stopUserStatisticsResetQueue(); + process.env.REDIS_URL = "redis://localhost:6379"; + boundary.processHandler = null; + boundary.failedHandler = null; + boundary.processName = null; + boundary.queueOptions = null; + for (const mock of [ + boundary.add, + boundary.getJob, + boundary.close, + boundary.claim, + boundary.getStatus, + boundary.setStatus, + boundary.deleteStatus, + boundary.release, + boundary.execute, + ]) + mock.mockReset(); + boundary.add.mockResolvedValue({ id: "job" }); + boundary.getJob.mockResolvedValue({ + id: "job", + getState: vi.fn().mockResolvedValue("waiting"), + }); + boundary.setStatus.mockResolvedValue(undefined); + boundary.deleteStatus.mockResolvedValue(undefined); + boundary.release.mockResolvedValue(undefined); + }); + + it("returns the existing active reset instead of enqueueing a competitor", async () => { + boundary.claim.mockResolvedValue({ acquired: false, resetId: existing.resetId }); + boundary.getStatus.mockResolvedValue(existing); + + await expect(enqueueUserStatisticsReset(42)).resolves.toEqual(existing); + expect(boundary.deleteStatus).toHaveBeenCalledTimes(1); + expect(boundary.add).not.toHaveBeenCalled(); + }); + + it("configures five attempts with exponential 30 second backoff", async () => { + boundary.claim.mockImplementation(async (_userId: number, resetId: string) => ({ + acquired: true, + resetId, + })); + + const queued = await enqueueUserStatisticsReset(42); + + expect(queued.status).toBe("queued"); + expect(boundary.queueOptions.defaultJobOptions).toMatchObject({ + attempts: 5, + backoff: { type: "exponential", delay: 30_000 }, + }); + expect(boundary.processName).toBe("reset"); + expect(boundary.add).toHaveBeenCalledWith( + "reset", + expect.objectContaining({ userId: 42, resetId: queued.resetId }), + { jobId: queued.resetId } + ); + }); + + it("recreates a missing Bull job for an existing queued claim", async () => { + const queued = { ...existing, status: "queued" as const, startedAt: null }; + boundary.claim.mockResolvedValue({ acquired: false, resetId: queued.resetId }); + boundary.getStatus.mockResolvedValue(queued); + boundary.getJob.mockResolvedValue(null); + + await expect(enqueueUserStatisticsReset(42)).resolves.toEqual(queued); + + expect(boundary.add).toHaveBeenCalledWith( + "reset", + { + resetId: queued.resetId, + userId: queued.userId, + requestedAt: queued.requestedAt, + }, + { jobId: queued.resetId } + ); + }); + + it("releases a stale terminal claim and creates a new reset", async () => { + boundary.claim + .mockResolvedValueOnce({ acquired: false, resetId: existing.resetId }) + .mockImplementationOnce(async (_userId: number, resetId: string) => ({ + acquired: true, + resetId, + })); + boundary.getStatus.mockResolvedValue({ ...existing, status: "completed" }); + + const queued = await enqueueUserStatisticsReset(42); + + expect(queued.status).toBe("queued"); + expect(queued.resetId).not.toBe(existing.resetId); + expect(boundary.release).toHaveBeenCalledWith(42, existing.resetId); + expect(boundary.add).toHaveBeenCalledWith( + "reset", + expect.objectContaining({ resetId: queued.resetId }), + { jobId: queued.resetId } + ); + }); + + it("releases a claim whose retained Bull job is already terminal", async () => { + boundary.claim + .mockResolvedValueOnce({ acquired: false, resetId: existing.resetId }) + .mockImplementationOnce(async (_userId: number, resetId: string) => ({ + acquired: true, + resetId, + })); + boundary.getStatus.mockResolvedValue({ ...existing, status: "running" }); + boundary.getJob.mockResolvedValue({ + id: existing.resetId, + getState: vi.fn().mockResolvedValue("failed"), + }); + + const queued = await enqueueUserStatisticsReset(42); + + expect(queued.resetId).not.toBe(existing.resetId); + expect(boundary.release).toHaveBeenCalledWith(42, existing.resetId); + }); + + it("does not fail application startup when Redis is not configured", async () => { + await stopUserStatisticsResetQueue(); + delete process.env.REDIS_URL; + + expect(startUserStatisticsResetQueue()).toBe(false); + expect(boundary.processHandler).toBeNull(); + }); + + it("moves a Bull job through running to completed and releases the active claim", async () => { + boundary.claim.mockImplementation(async (_userId: number, resetId: string) => ({ + acquired: true, + resetId, + })); + const queued = await enqueueUserStatisticsReset(42); + boundary.getStatus.mockResolvedValue(queued); + boundary.execute.mockResolvedValue({ + deletedMessageRequests: 2000, + deletedUsageLedger: 1500, + }); + + await boundary.processHandler?.({ + data: queued, + attemptsMade: 0, + opts: { attempts: 5 }, + }); + + expect(boundary.setStatus).toHaveBeenLastCalledWith( + expect.objectContaining({ + status: "completed", + deletedMessageRequests: 2000, + deletedUsageLedger: 1500, + }) + ); + expect(boundary.release).toHaveBeenCalledWith(42, queued.resetId); + }); + + it("keeps retryable failures active and records a stable final error", async () => { + boundary.claim.mockImplementation(async (_userId: number, resetId: string) => ({ + acquired: true, + resetId, + })); + const queued = await enqueueUserStatisticsReset(42); + boundary.getStatus.mockResolvedValue(queued); + boundary.execute.mockRejectedValue(new Error("database timeout")); + + await expect( + boundary.processHandler?.({ data: queued, attemptsMade: 0, opts: { attempts: 5 } }) + ).rejects.toThrow("database timeout"); + expect(boundary.setStatus).toHaveBeenLastCalledWith( + expect.objectContaining({ status: "queued", errorCode: null }) + ); + expect(boundary.release).not.toHaveBeenCalled(); + + boundary.setStatus.mockClear(); + await expect( + boundary.processHandler?.({ data: queued, attemptsMade: 4, opts: { attempts: 5 } }) + ).rejects.toThrow("database timeout"); + expect(boundary.setStatus).toHaveBeenLastCalledWith( + expect.objectContaining({ + status: "failed", + errorCode: "USER_STATISTICS_RESET_OPERATION_FAILED", + }) + ); + expect(boundary.release).toHaveBeenCalledWith(42, queued.resetId); + }); + + it("marks max-stalled jobs failed even when attemptsMade did not advance", async () => { + startUserStatisticsResetQueue(); + boundary.getStatus.mockResolvedValue(existing); + + await boundary.failedHandler?.( + { + data: existing, + attemptsMade: 0, + opts: { attempts: 5 }, + }, + new Error("job stalled more than allowable limit") + ); + + expect(boundary.setStatus).toHaveBeenLastCalledWith( + expect.objectContaining({ + status: "failed", + errorCode: "USER_STATISTICS_RESET_OPERATION_FAILED", + }) + ); + expect(boundary.release).toHaveBeenCalledWith(42, existing.resetId); + }); +}); diff --git a/tests/unit/lib/user-statistics-reset-service.test.ts b/tests/unit/lib/user-statistics-reset-service.test.ts new file mode 100644 index 000000000..43e9406ee --- /dev/null +++ b/tests/unit/lib/user-statistics-reset-service.test.ts @@ -0,0 +1,174 @@ +import type { SQL } from "drizzle-orm"; +import { PgDialect } from "drizzle-orm/pg-core"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const boundary = vi.hoisted(() => ({ + transactionResults: [] as unknown[], + transactionQueries: [] as unknown[], + executeResults: [] as unknown[], + executeQueries: [] as unknown[], + userKeys: [] as { id: number; key: string }[], + cacheResult: { costKeysDeleted: 0, activeSessionsDeleted: 0, durationMs: 1 } as unknown, + clearUserCostCache: vi.fn(), + invalidateCachedUser: vi.fn(), + updateSet: null as Record | null, + updateError: null as Error | null, +})); + +vi.mock("@/drizzle/db", () => ({ + db: { + transaction: async ( + callback: (tx: { execute: (query: unknown) => Promise }) => unknown + ) => + callback({ + execute: async (query: unknown) => { + boundary.transactionQueries.push(query); + const result = boundary.transactionResults.shift(); + if (result instanceof Error) throw result; + return result; + }, + }), + execute: async (query: unknown) => { + boundary.executeQueries.push(query); + const result = boundary.executeResults.shift(); + if (result instanceof Error) throw result; + return result; + }, + select: () => ({ from: () => ({ where: async () => boundary.userKeys }) }), + update: () => ({ + set: (value: Record) => { + boundary.updateSet = value; + return { + where: async () => { + if (boundary.updateError) throw boundary.updateError; + }, + }; + }, + }), + }, +})); +vi.mock("@/lib/redis/cost-cache-cleanup", () => ({ + clearUserCostCache: boundary.clearUserCostCache, +})); +vi.mock("@/lib/security/api-key-auth-cache", () => ({ + invalidateCachedUser: boundary.invalidateCachedUser, +})); + +import { + executeUserStatisticsReset, + type UserStatisticsResetError, +} from "@/lib/user-statistics-reset/reset-service"; + +const dialect = new PgDialect(); +const sqlText = (query: unknown) => dialect.sqlToQuery(query as SQL).sql.toLowerCase(); + +describe("executeUserStatisticsReset", () => { + beforeEach(() => { + boundary.transactionResults = []; + boundary.transactionQueries = []; + boundary.executeResults = []; + boundary.executeQueries = []; + boundary.userKeys = [{ id: 9, key: "key-hash" }]; + boundary.updateSet = null; + boundary.updateError = null; + boundary.clearUserCostCache.mockReset().mockResolvedValue(boundary.cacheResult); + boundary.invalidateCachedUser.mockReset().mockResolvedValue(undefined); + }); + + it("deletes both tables in independent cutoff batches and preserves active sessions", async () => { + boundary.transactionResults = [{ count: 1000 }, { count: 7 }, { count: 3 }]; + boundary.executeResults = [[{ exists: false }], [{ exists: false }]]; + + await expect( + executeUserStatisticsReset({ userId: 42, requestedAt: "2026-08-02T12:00:00.000Z" }) + ).resolves.toEqual({ deletedMessageRequests: 1007, deletedUsageLedger: 3 }); + + expect(boundary.transactionQueries).toHaveLength(3); + expect(sqlText(boundary.transactionQueries[0])).toContain("from message_request"); + expect(sqlText(boundary.transactionQueries[0])).toContain("created_at is null"); + expect(sqlText(boundary.transactionQueries[0])).toContain("created_at <= $2"); + expect(sqlText(boundary.transactionQueries[0])).toContain("limit $3"); + expect(sqlText(boundary.transactionQueries[0])).toContain("for update skip locked"); + expect(sqlText(boundary.transactionQueries[2])).toContain("from usage_ledger"); + expect(boundary.clearUserCostCache).toHaveBeenCalledWith({ + userId: 42, + keyIds: [9], + keyHashes: ["key-hash"], + includeActiveSessions: false, + allowWhenRateLimitDisabled: true, + }); + expect(sqlText(boundary.updateSet?.costResetAt)).toContain("case when"); + expect(sqlText(boundary.updateSet?.limit5hCostResetAt)).toContain("case when"); + }); + + it("reports deleted rows when cache cleanup fails so retries preserve progress", async () => { + boundary.transactionResults = [{ count: 4 }, { count: 6 }]; + boundary.executeResults = [[{ exists: false }], [{ exists: false }]]; + boundary.clearUserCostCache.mockResolvedValue({ cleanupFailed: true }); + + await expect( + executeUserStatisticsReset({ userId: 42, requestedAt: "2026-08-02T12:00:00.000Z" }) + ).rejects.toEqual( + expect.objectContaining({ + code: "USER_STATISTICS_RESET_CACHE_CLEANUP_FAILED", + progress: { deletedMessageRequests: 4, deletedUsageLedger: 6 }, + }) + ); + }); + + it("fails retryably when an empty batch still has locked cutoff rows", async () => { + boundary.transactionResults = [{ count: 0 }]; + boundary.executeResults = [[{ exists: true }]]; + + await expect( + executeUserStatisticsReset({ userId: 42, requestedAt: "2026-08-02T12:00:00.000Z" }) + ).rejects.toEqual( + expect.objectContaining({ + code: "USER_STATISTICS_RESET_ROWS_LOCKED", + }) + ); + expect(boundary.clearUserCostCache).not.toHaveBeenCalled(); + }); + + it("preserves completed message deletion progress when ledger deletion throws", async () => { + boundary.transactionResults = [{ count: 4 }, new Error("ledger unavailable")]; + boundary.executeResults = [[{ exists: false }]]; + + await expect( + executeUserStatisticsReset({ userId: 42, requestedAt: "2026-08-02T12:00:00.000Z" }) + ).rejects.toEqual( + expect.objectContaining({ + code: "USER_STATISTICS_RESET_OPERATION_FAILED", + progress: { deletedMessageRequests: 4, deletedUsageLedger: 0 }, + }) + ); + }); + + it("preserves progress when a later batch fails", async () => { + boundary.transactionResults = [{ count: 1000 }, new Error("statement timeout")]; + + await expect( + executeUserStatisticsReset({ userId: 42, requestedAt: "2026-08-02T12:00:00.000Z" }) + ).rejects.toEqual( + expect.objectContaining({ + code: "USER_STATISTICS_RESET_OPERATION_FAILED", + progress: { deletedMessageRequests: 1000, deletedUsageLedger: 0 }, + }) + ); + }); + + it("preserves both table counts when marker cleanup fails", async () => { + boundary.transactionResults = [{ count: 4 }, { count: 6 }]; + boundary.executeResults = [[{ exists: false }], [{ exists: false }]]; + boundary.updateError = new Error("database unavailable"); + + await expect( + executeUserStatisticsReset({ userId: 42, requestedAt: "2026-08-02T12:00:00.000Z" }) + ).rejects.toEqual( + expect.objectContaining({ + code: "USER_STATISTICS_RESET_OPERATION_FAILED", + progress: { deletedMessageRequests: 4, deletedUsageLedger: 6 }, + }) + ); + }); +}); diff --git a/tests/unit/lib/user-statistics-reset-status-store.test.ts b/tests/unit/lib/user-statistics-reset-status-store.test.ts new file mode 100644 index 000000000..7ea16b9c2 --- /dev/null +++ b/tests/unit/lib/user-statistics-reset-status-store.test.ts @@ -0,0 +1,128 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const boundary = vi.hoisted(() => ({ + redis: { + status: "ready", + get: vi.fn(), + set: vi.fn(), + del: vi.fn(), + eval: vi.fn(), + }, + storeSet: vi.fn(), +})); + +vi.mock("@/lib/redis/client", () => ({ + getRedisClient: () => boundary.redis, +})); + +vi.mock("@/lib/redis/redis-kv-store", () => ({ + RedisKVStore: class MockRedisKVStore { + set = boundary.storeSet; + }, +})); + +import { + claimActiveUserStatisticsReset, + deleteUserStatisticsResetStatus, + getUserStatisticsResetStatus, + releaseActiveUserStatisticsReset, + setUserStatisticsResetStatus, +} from "@/lib/user-statistics-reset/reset-status-store"; + +const record = { + resetId: "00000000-0000-4000-8000-000000000001", + userId: 42, + status: "queued" as const, + requestedAt: "2026-08-02T12:00:00.000Z", + startedAt: null, + completedAt: null, + deletedMessageRequests: 0, + deletedUsageLedger: 0, + errorCode: null, +}; + +describe("user statistics reset status store", () => { + beforeEach(() => { + boundary.redis.status = "ready"; + for (const mock of [ + boundary.redis.get, + boundary.redis.set, + boundary.redis.del, + boundary.redis.eval, + boundary.storeSet, + ]) { + mock.mockReset(); + } + boundary.storeSet.mockResolvedValue(true); + }); + + it("writes status records and fails closed when Redis rejects the write", async () => { + await expect(setUserStatisticsResetStatus(record)).resolves.toBeUndefined(); + expect(boundary.storeSet).toHaveBeenCalledWith(record.resetId, record); + + boundary.storeSet.mockResolvedValue(false); + await expect(setUserStatisticsResetStatus(record)).rejects.toThrow( + "USER_STATISTICS_RESET_STATUS_WRITE_FAILED" + ); + }); + + it("reads valid records and distinguishes missing, corrupt, and unavailable state", async () => { + boundary.redis.get.mockResolvedValueOnce(JSON.stringify(record)).mockResolvedValueOnce(null); + + await expect(getUserStatisticsResetStatus(record.resetId)).resolves.toEqual(record); + await expect(getUserStatisticsResetStatus(record.resetId)).resolves.toBeNull(); + + boundary.redis.get.mockResolvedValue("not-json"); + await expect(getUserStatisticsResetStatus(record.resetId)).rejects.toThrow( + "USER_STATISTICS_RESET_STATUS_INVALID" + ); + + boundary.redis.status = "connecting"; + await expect(getUserStatisticsResetStatus(record.resetId)).rejects.toThrow( + "USER_STATISTICS_RESET_REDIS_UNAVAILABLE" + ); + }); + + it("claims one active reset and returns the existing owner on contention", async () => { + boundary.redis.set.mockResolvedValueOnce("OK").mockResolvedValueOnce(null); + boundary.redis.get.mockResolvedValue(record.resetId); + + await expect(claimActiveUserStatisticsReset(42, record.resetId)).resolves.toEqual({ + acquired: true, + resetId: record.resetId, + }); + await expect(claimActiveUserStatisticsReset(42, "new-reset")).resolves.toEqual({ + acquired: false, + resetId: record.resetId, + }); + expect(boundary.redis.set).toHaveBeenCalledWith( + "cch:user-statistics-reset:active:42", + record.resetId, + "EX", + 604_800, + "NX" + ); + }); + + it("fails contention without an owner and deletes status or claims by exact key", async () => { + boundary.redis.set.mockResolvedValue(null); + boundary.redis.get.mockResolvedValue(null); + + await expect(claimActiveUserStatisticsReset(42, record.resetId)).rejects.toThrow( + "USER_STATISTICS_RESET_ACTIVE_CLAIM_FAILED" + ); + + await deleteUserStatisticsResetStatus(record.resetId); + await releaseActiveUserStatisticsReset(42, record.resetId); + + expect(boundary.redis.del).toHaveBeenCalledWith( + `cch:user-statistics-reset:status:${record.resetId}` + ); + expect(boundary.redis.eval).toHaveBeenCalledWith( + expect.stringContaining("redis.call('GET'"), + 1, + "cch:user-statistics-reset:active:42", + record.resetId + ); + }); +}); From 6e5f8ab075d0908a2b902756682f68208d0fe6b6 Mon Sep 17 00:00:00 2001 From: ding113 Date: Mon, 3 Aug 2026 01:58:32 +0800 Subject: [PATCH 05/18] fix(migrations): roll out timeout indexes concurrently --- drizzle/0118_bright_sunspot.sql | 5 + drizzle/meta/0118_snapshot.json | 5397 +++++++++++++++++ drizzle/meta/_journal.json | 7 + src/drizzle/schema.ts | 23 +- src/lib/migrate.ts | 1 + .../session-replay-index-preflight.ts | 69 +- .../database-timeout-migration.test.ts | 41 + .../unit/drizzle/proxy-status-indexes.test.ts | 37 + .../drizzle/session-identity-indexes.test.ts | 27 + .../session-replay-index-preflight.test.ts | 121 +- 10 files changed, 5695 insertions(+), 33 deletions(-) create mode 100644 drizzle/0118_bright_sunspot.sql create mode 100644 drizzle/meta/0118_snapshot.json create mode 100644 tests/unit/drizzle/database-timeout-migration.test.ts create mode 100644 tests/unit/drizzle/proxy-status-indexes.test.ts diff --git a/drizzle/0118_bright_sunspot.sql b/drizzle/0118_bright_sunspot.sql new file mode 100644 index 000000000..0456be602 --- /dev/null +++ b/drizzle/0118_bright_sunspot.sql @@ -0,0 +1,5 @@ +CREATE INDEX IF NOT EXISTS "idx_message_request_proxy_status_active" ON "message_request" USING btree ("created_at" DESC NULLS LAST,"user_id") WHERE "message_request"."deleted_at" IS NULL AND "message_request"."is_replay" = false AND "message_request"."status_code" IS NULL AND ("message_request"."blocked_by" IS NULL OR "message_request"."blocked_by" <> 'warmup');--> statement-breakpoint +CREATE INDEX IF NOT EXISTS "idx_message_request_proxy_status_latest" ON "message_request" USING btree ("user_id","updated_at" DESC NULLS LAST,"id" DESC NULLS LAST) WHERE "message_request"."deleted_at" IS NULL AND "message_request"."is_replay" = false AND "message_request"."status_code" IS NOT NULL AND ("message_request"."blocked_by" IS NULL OR "message_request"."blocked_by" <> 'warmup');--> statement-breakpoint +CREATE INDEX IF NOT EXISTS "idx_usage_ledger_user_id_reset" ON "usage_ledger" USING btree ("user_id");--> statement-breakpoint +CREATE INDEX IF NOT EXISTS "idx_message_request_session_identity_created_at" ON "message_request" USING btree (COALESCE("session_identity", "session_id"),"created_at" DESC NULLS LAST,"id" DESC NULLS LAST) WHERE "message_request"."deleted_at" IS NULL;--> statement-breakpoint +CREATE INDEX IF NOT EXISTS "idx_usage_ledger_session_identity_created_at" ON "usage_ledger" USING btree (COALESCE("session_identity", "session_id"),"user_id","created_at" DESC NULLS LAST) WHERE "usage_ledger"."blocked_by" IS NULL AND "usage_ledger"."is_replay" = false; diff --git a/drizzle/meta/0118_snapshot.json b/drizzle/meta/0118_snapshot.json new file mode 100644 index 000000000..c828516f4 --- /dev/null +++ b/drizzle/meta/0118_snapshot.json @@ -0,0 +1,5397 @@ +{ + "id": "0a61db74-24d2-408d-b247-974f2413fa50", + "prevId": "f20d88c1-dfee-4399-ae01-147936dc1177", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.audit_log": { + "name": "audit_log", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "action_category": { + "name": "action_category", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true + }, + "action_type": { + "name": "action_type", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true + }, + "target_type": { + "name": "target_type", + "type": "varchar(32)", + "primaryKey": false, + "notNull": false + }, + "target_id": { + "name": "target_id", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "target_name": { + "name": "target_name", + "type": "varchar(256)", + "primaryKey": false, + "notNull": false + }, + "before_value": { + "name": "before_value", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "after_value": { + "name": "after_value", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "operator_user_id": { + "name": "operator_user_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "operator_user_name": { + "name": "operator_user_name", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false + }, + "operator_key_id": { + "name": "operator_key_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "operator_key_name": { + "name": "operator_key_name", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false + }, + "operator_ip": { + "name": "operator_ip", + "type": "varchar(45)", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false + }, + "success": { + "name": "success", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_audit_log_category_created_at": { + "name": "idx_audit_log_category_created_at", + "columns": [ + { + "expression": "action_category", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_audit_log_operator_user_created_at": { + "name": "idx_audit_log_operator_user_created_at", + "columns": [ + { + "expression": "operator_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"audit_log\".\"operator_user_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_audit_log_operator_ip_created_at": { + "name": "idx_audit_log_operator_ip_created_at", + "columns": [ + { + "expression": "operator_ip", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"audit_log\".\"operator_ip\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_audit_log_target": { + "name": "idx_audit_log_target", + "columns": [ + { + "expression": "target_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"audit_log\".\"target_type\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_audit_log_created_at_id": { + "name": "idx_audit_log_created_at_id", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.cloud_pricing_catalog": { + "name": "cloud_pricing_catalog", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "version": { + "name": "version", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true + }, + "currency": { + "name": "currency", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "default": "'USD'" + }, + "refreshed_at": { + "name": "refreshed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "providers": { + "name": "providers", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "vendors": { + "name": "vendors", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "model_count": { + "name": "model_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "synced_at": { + "name": "synced_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.error_rules": { + "name": "error_rules", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "pattern": { + "name": "pattern", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "match_type": { + "name": "match_type", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'regex'" + }, + "category": { + "name": "category", + "type": "varchar(50)", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "override_response": { + "name": "override_response", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "override_status_code": { + "name": "override_status_code", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "is_enabled": { + "name": "is_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "is_default": { + "name": "is_default", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "priority": { + "name": "priority", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "idx_error_rules_enabled": { + "name": "idx_error_rules_enabled", + "columns": [ + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "priority", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "unique_pattern": { + "name": "unique_pattern", + "columns": [ + { + "expression": "pattern", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_category": { + "name": "idx_category", + "columns": [ + { + "expression": "category", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_match_type": { + "name": "idx_match_type", + "columns": [ + { + "expression": "match_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.keys": { + "name": "keys", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "is_enabled": { + "name": "is_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "can_login_web_ui": { + "name": "can_login_web_ui", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "limit_5h_usd": { + "name": "limit_5h_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "limit_5h_reset_mode": { + "name": "limit_5h_reset_mode", + "type": "daily_reset_mode", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'rolling'" + }, + "limit_daily_usd": { + "name": "limit_daily_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "daily_reset_mode": { + "name": "daily_reset_mode", + "type": "daily_reset_mode", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'fixed'" + }, + "daily_reset_time": { + "name": "daily_reset_time", + "type": "varchar(5)", + "primaryKey": false, + "notNull": true, + "default": "'00:00'" + }, + "limit_weekly_usd": { + "name": "limit_weekly_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "limit_monthly_usd": { + "name": "limit_monthly_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "limit_total_usd": { + "name": "limit_total_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "cost_reset_at": { + "name": "cost_reset_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "limit_concurrent_sessions": { + "name": "limit_concurrent_sessions", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "provider_group": { + "name": "provider_group", + "type": "varchar(200)", + "primaryKey": false, + "notNull": false, + "default": "'default'" + }, + "cache_ttl_preference": { + "name": "cache_ttl_preference", + "type": "varchar(10)", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "idx_keys_user_id": { + "name": "idx_keys_user_id", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_keys_key": { + "name": "idx_keys_key", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_keys_created_at": { + "name": "idx_keys_created_at", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_keys_deleted_at": { + "name": "idx_keys_deleted_at", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.message_request": { + "name": "message_request", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "model": { + "name": "model", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "cost_usd": { + "name": "cost_usd", + "type": "numeric(21, 15)", + "primaryKey": false, + "notNull": false, + "default": "'0'" + }, + "cost_multiplier": { + "name": "cost_multiplier", + "type": "numeric(10, 4)", + "primaryKey": false, + "notNull": false + }, + "group_cost_multiplier": { + "name": "group_cost_multiplier", + "type": "numeric(10, 4)", + "primaryKey": false, + "notNull": false + }, + "cost_breakdown": { + "name": "cost_breakdown", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "session_id": { + "name": "session_id", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "session_identity": { + "name": "session_identity", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "session_identity_kind": { + "name": "session_identity_kind", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "affinity_scope_tag": { + "name": "affinity_scope_tag", + "type": "varchar(16)", + "primaryKey": false, + "notNull": false + }, + "affinity_fingerprint": { + "name": "affinity_fingerprint", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "affinity_fingerprint_chain": { + "name": "affinity_fingerprint_chain", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "is_replay": { + "name": "is_replay", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "replay_source_request_id": { + "name": "replay_source_request_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "request_sequence": { + "name": "request_sequence", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 1 + }, + "provider_chain": { + "name": "provider_chain", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "routing_trace": { + "name": "routing_trace", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "status_code": { + "name": "status_code", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "api_type": { + "name": "api_type", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "endpoint": { + "name": "endpoint", + "type": "varchar(256)", + "primaryKey": false, + "notNull": false + }, + "original_model": { + "name": "original_model", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false + }, + "actual_response_model": { + "name": "actual_response_model", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false + }, + "input_tokens": { + "name": "input_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "output_tokens": { + "name": "output_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "ttfb_ms": { + "name": "ttfb_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "first_byte_ms": { + "name": "first_byte_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "cache_creation_input_tokens": { + "name": "cache_creation_input_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "cache_read_input_tokens": { + "name": "cache_read_input_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "cache_creation_5m_input_tokens": { + "name": "cache_creation_5m_input_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "cache_creation_1h_input_tokens": { + "name": "cache_creation_1h_input_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "cache_ttl_applied": { + "name": "cache_ttl_applied", + "type": "varchar(10)", + "primaryKey": false, + "notNull": false + }, + "context_1m_applied": { + "name": "context_1m_applied", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "swap_cache_ttl_applied": { + "name": "swap_cache_ttl_applied", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "special_settings": { + "name": "special_settings", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "hedge_losers": { + "name": "hedge_losers", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error_stack": { + "name": "error_stack", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error_cause": { + "name": "error_cause", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "blocked_by": { + "name": "blocked_by", + "type": "varchar(50)", + "primaryKey": false, + "notNull": false + }, + "blocked_reason": { + "name": "blocked_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false + }, + "client_ip": { + "name": "client_ip", + "type": "varchar(45)", + "primaryKey": false, + "notNull": false + }, + "messages_count": { + "name": "messages_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "cache_compatibility_key": { + "name": "cache_compatibility_key", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "cache_score_eligible": { + "name": "cache_score_eligible", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "cache_score_excluded_reason": { + "name": "cache_score_excluded_reason", + "type": "varchar(32)", + "primaryKey": false, + "notNull": false + }, + "theoretical_cache_tokens": { + "name": "theoretical_cache_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "cache_ttl_bucket": { + "name": "cache_ttl_bucket", + "type": "varchar(10)", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "idx_message_request_user_date_cost": { + "name": "idx_message_request_user_date_cost", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "cost_usd", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_user_created_at_cost_stats": { + "name": "idx_message_request_user_created_at_cost_stats", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "cost_usd", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND (\"message_request\".\"blocked_by\" IS NULL OR \"message_request\".\"blocked_by\" <> 'warmup')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_user_query": { + "name": "idx_message_request_user_query", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_provider_created_at_active": { + "name": "idx_message_request_provider_created_at_active", + "columns": [ + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND (\"message_request\".\"blocked_by\" IS NULL OR \"message_request\".\"blocked_by\" <> 'warmup')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_provider_created_at_finalized_active": { + "name": "idx_message_request_provider_created_at_finalized_active", + "columns": [ + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND \"message_request\".\"status_code\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_proxy_status_active": { + "name": "idx_message_request_proxy_status_active", + "columns": [ + { + "expression": "\"created_at\" DESC NULLS LAST", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND \"message_request\".\"is_replay\" = false AND \"message_request\".\"status_code\" IS NULL AND (\"message_request\".\"blocked_by\" IS NULL OR \"message_request\".\"blocked_by\" <> 'warmup')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_proxy_status_latest": { + "name": "idx_message_request_proxy_status_latest", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "\"updated_at\" DESC NULLS LAST", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND \"message_request\".\"is_replay\" = false AND \"message_request\".\"status_code\" IS NOT NULL AND (\"message_request\".\"blocked_by\" IS NULL OR \"message_request\".\"blocked_by\" <> 'warmup')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_session_id": { + "name": "idx_message_request_session_id", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_session_id_prefix": { + "name": "idx_message_request_session_id_prefix", + "columns": [ + { + "expression": "\"session_id\" varchar_pattern_ops", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND (\"message_request\".\"blocked_by\" IS NULL OR \"message_request\".\"blocked_by\" <> 'warmup')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_session_seq": { + "name": "idx_message_request_session_seq", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "request_sequence", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_session_identity_created_at": { + "name": "idx_message_request_session_identity_created_at", + "columns": [ + { + "expression": "COALESCE(\"session_identity\", \"session_id\")", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "\"created_at\" DESC NULLS LAST", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_endpoint": { + "name": "idx_message_request_endpoint", + "columns": [ + { + "expression": "endpoint", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_blocked_by": { + "name": "idx_message_request_blocked_by", + "columns": [ + { + "expression": "blocked_by", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_provider_id": { + "name": "idx_message_request_provider_id", + "columns": [ + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_user_id": { + "name": "idx_message_request_user_id", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_key": { + "name": "idx_message_request_key", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_key_created_at_id": { + "name": "idx_message_request_key_created_at_id", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_key_model_active": { + "name": "idx_message_request_key_model_active", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "model", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND \"message_request\".\"model\" IS NOT NULL AND (\"message_request\".\"blocked_by\" IS NULL OR \"message_request\".\"blocked_by\" <> 'warmup')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_key_endpoint_active": { + "name": "idx_message_request_key_endpoint_active", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "endpoint", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND \"message_request\".\"endpoint\" IS NOT NULL AND (\"message_request\".\"blocked_by\" IS NULL OR \"message_request\".\"blocked_by\" <> 'warmup')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_created_at_id_active": { + "name": "idx_message_request_created_at_id_active", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_model_active": { + "name": "idx_message_request_model_active", + "columns": [ + { + "expression": "model", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND \"message_request\".\"model\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_status_code_active": { + "name": "idx_message_request_status_code_active", + "columns": [ + { + "expression": "status_code", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND \"message_request\".\"status_code\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_created_at": { + "name": "idx_message_request_created_at", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_deleted_at": { + "name": "idx_message_request_deleted_at", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_key_last_active": { + "name": "idx_message_request_key_last_active", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND (\"message_request\".\"blocked_by\" IS NULL OR \"message_request\".\"blocked_by\" <> 'warmup')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_key_cost_active": { + "name": "idx_message_request_key_cost_active", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "cost_usd", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND (\"message_request\".\"blocked_by\" IS NULL OR \"message_request\".\"blocked_by\" <> 'warmup')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_session_user_info": { + "name": "idx_message_request_session_user_info", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_client_ip_created_at": { + "name": "idx_message_request_client_ip_created_at", + "columns": [ + { + "expression": "client_ip", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND \"message_request\".\"client_ip\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.model_prices": { + "name": "model_prices", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "model_name": { + "name": "model_name", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "price_data": { + "name": "price_data", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'cloud'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "idx_model_prices_latest": { + "name": "idx_model_prices_latest", + "columns": [ + { + "expression": "model_name", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_model_prices_model_name": { + "name": "idx_model_prices_model_name", + "columns": [ + { + "expression": "model_name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_model_prices_created_at": { + "name": "idx_model_prices_created_at", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_model_prices_source": { + "name": "idx_model_prices_source", + "columns": [ + { + "expression": "source", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_model_prices_vendor": { + "name": "idx_model_prices_vendor", + "columns": [ + { + "expression": "((\"price_data\" ->> 'vendor'))", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_model_prices_aliases": { + "name": "idx_model_prices_aliases", + "columns": [ + { + "expression": "((\"price_data\" -> 'aliases'))", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.notification_settings": { + "name": "notification_settings", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "use_legacy_mode": { + "name": "use_legacy_mode", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "circuit_breaker_enabled": { + "name": "circuit_breaker_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "circuit_breaker_webhook": { + "name": "circuit_breaker_webhook", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false + }, + "daily_leaderboard_enabled": { + "name": "daily_leaderboard_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "daily_leaderboard_webhook": { + "name": "daily_leaderboard_webhook", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false + }, + "daily_leaderboard_time": { + "name": "daily_leaderboard_time", + "type": "varchar(10)", + "primaryKey": false, + "notNull": false, + "default": "'09:00'" + }, + "daily_leaderboard_top_n": { + "name": "daily_leaderboard_top_n", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 5 + }, + "cost_alert_enabled": { + "name": "cost_alert_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "cost_alert_webhook": { + "name": "cost_alert_webhook", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false + }, + "cost_alert_threshold": { + "name": "cost_alert_threshold", + "type": "numeric(5, 2)", + "primaryKey": false, + "notNull": false, + "default": "'0.80'" + }, + "cost_alert_check_interval": { + "name": "cost_alert_check_interval", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 60 + }, + "cache_hit_rate_alert_enabled": { + "name": "cache_hit_rate_alert_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "cache_hit_rate_alert_webhook": { + "name": "cache_hit_rate_alert_webhook", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false + }, + "cache_hit_rate_alert_window_mode": { + "name": "cache_hit_rate_alert_window_mode", + "type": "varchar(10)", + "primaryKey": false, + "notNull": false, + "default": "'auto'" + }, + "cache_hit_rate_alert_check_interval": { + "name": "cache_hit_rate_alert_check_interval", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 5 + }, + "cache_hit_rate_alert_historical_lookback_days": { + "name": "cache_hit_rate_alert_historical_lookback_days", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 7 + }, + "cache_hit_rate_alert_min_eligible_requests": { + "name": "cache_hit_rate_alert_min_eligible_requests", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 20 + }, + "cache_hit_rate_alert_min_eligible_tokens": { + "name": "cache_hit_rate_alert_min_eligible_tokens", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "cache_hit_rate_alert_abs_min": { + "name": "cache_hit_rate_alert_abs_min", + "type": "numeric(5, 4)", + "primaryKey": false, + "notNull": false, + "default": "'0.05'" + }, + "cache_hit_rate_alert_drop_rel": { + "name": "cache_hit_rate_alert_drop_rel", + "type": "numeric(5, 4)", + "primaryKey": false, + "notNull": false, + "default": "'0.3'" + }, + "cache_hit_rate_alert_drop_abs": { + "name": "cache_hit_rate_alert_drop_abs", + "type": "numeric(5, 4)", + "primaryKey": false, + "notNull": false, + "default": "'0.1'" + }, + "cache_hit_rate_alert_cooldown_minutes": { + "name": "cache_hit_rate_alert_cooldown_minutes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 30 + }, + "cache_hit_rate_alert_top_n": { + "name": "cache_hit_rate_alert_top_n", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 10 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.notification_target_bindings": { + "name": "notification_target_bindings", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "notification_type": { + "name": "notification_type", + "type": "notification_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "target_id": { + "name": "target_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "is_enabled": { + "name": "is_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "schedule_cron": { + "name": "schedule_cron", + "type": "varchar(100)", + "primaryKey": false, + "notNull": false + }, + "schedule_timezone": { + "name": "schedule_timezone", + "type": "varchar(50)", + "primaryKey": false, + "notNull": false + }, + "template_override": { + "name": "template_override", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "unique_notification_target_binding": { + "name": "unique_notification_target_binding", + "columns": [ + { + "expression": "notification_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_notification_bindings_type": { + "name": "idx_notification_bindings_type", + "columns": [ + { + "expression": "notification_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_notification_bindings_target": { + "name": "idx_notification_bindings_target", + "columns": [ + { + "expression": "target_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "notification_target_bindings_target_id_webhook_targets_id_fk": { + "name": "notification_target_bindings_target_id_webhook_targets_id_fk", + "tableFrom": "notification_target_bindings", + "tableTo": "webhook_targets", + "columnsFrom": [ + "target_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.provider_batch_apply_operations": { + "name": "provider_batch_apply_operations", + "schema": "", + "columns": { + "claim_key": { + "name": "claim_key", + "type": "varchar(256)", + "primaryKey": true, + "notNull": true + }, + "preview_token": { + "name": "preview_token", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "payload_fingerprint": { + "name": "payload_fingerprint", + "type": "varchar(128)", + "primaryKey": false, + "notNull": true + }, + "operation_id": { + "name": "operation_id", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "undo_token": { + "name": "undo_token", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "undo_expires_at": { + "name": "undo_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "undo_consumed_at": { + "name": "undo_consumed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true + }, + "result": { + "name": "result", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "uniq_provider_batch_apply_operations_preview_token": { + "name": "uniq_provider_batch_apply_operations_preview_token", + "columns": [ + { + "expression": "preview_token", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "uniq_provider_batch_apply_operations_operation_id": { + "name": "uniq_provider_batch_apply_operations_operation_id", + "columns": [ + { + "expression": "operation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "uniq_provider_batch_apply_operations_undo_token": { + "name": "uniq_provider_batch_apply_operations_undo_token", + "columns": [ + { + "expression": "undo_token", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_provider_batch_apply_operations_expires_at": { + "name": "idx_provider_batch_apply_operations_expires_at", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.provider_cache_effectiveness": { + "name": "provider_cache_effectiveness", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "model": { + "name": "model", + "type": "varchar(128)", + "primaryKey": false, + "notNull": true + }, + "cache_ttl_bucket": { + "name": "cache_ttl_bucket", + "type": "varchar(10)", + "primaryKey": false, + "notNull": true + }, + "window_start": { + "name": "window_start", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "window_end": { + "name": "window_end", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "sample_count": { + "name": "sample_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "eligible_count": { + "name": "eligible_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "theoretical_cache_tokens": { + "name": "theoretical_cache_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "observed_cache_read_tokens": { + "name": "observed_cache_read_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "raw_effectiveness_bp": { + "name": "raw_effectiveness_bp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "confidence_bp": { + "name": "confidence_bp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "effectiveness_bp": { + "name": "effectiveness_bp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "idx_provider_cache_effectiveness_window": { + "name": "idx_provider_cache_effectiveness_window", + "columns": [ + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "model", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "window_start", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.provider_endpoint_probe_logs": { + "name": "provider_endpoint_probe_logs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "endpoint_id": { + "name": "endpoint_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'scheduled'" + }, + "ok": { + "name": "ok", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "status_code": { + "name": "status_code", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "latency_ms": { + "name": "latency_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "error_type": { + "name": "error_type", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "idx_provider_endpoint_probe_logs_endpoint_created_at": { + "name": "idx_provider_endpoint_probe_logs_endpoint_created_at", + "columns": [ + { + "expression": "endpoint_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_provider_endpoint_probe_logs_created_at": { + "name": "idx_provider_endpoint_probe_logs_created_at", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "provider_endpoint_probe_logs_endpoint_id_provider_endpoints_id_fk": { + "name": "provider_endpoint_probe_logs_endpoint_id_provider_endpoints_id_fk", + "tableFrom": "provider_endpoint_probe_logs", + "tableTo": "provider_endpoints", + "columnsFrom": [ + "endpoint_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.provider_endpoints": { + "name": "provider_endpoints", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "vendor_id": { + "name": "vendor_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "provider_type": { + "name": "provider_type", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'claude'" + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "label": { + "name": "label", + "type": "varchar(200)", + "primaryKey": false, + "notNull": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "is_enabled": { + "name": "is_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "last_probed_at": { + "name": "last_probed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_probe_ok": { + "name": "last_probe_ok", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "last_probe_status_code": { + "name": "last_probe_status_code", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "last_probe_latency_ms": { + "name": "last_probe_latency_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "last_probe_error_type": { + "name": "last_probe_error_type", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "last_probe_error_message": { + "name": "last_probe_error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "uniq_provider_endpoints_vendor_type_url": { + "name": "uniq_provider_endpoints_vendor_type_url", + "columns": [ + { + "expression": "vendor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "url", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"provider_endpoints\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_provider_endpoints_vendor_type": { + "name": "idx_provider_endpoints_vendor_type", + "columns": [ + { + "expression": "vendor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"provider_endpoints\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_provider_endpoints_enabled": { + "name": "idx_provider_endpoints_enabled", + "columns": [ + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "vendor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"provider_endpoints\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_provider_endpoints_pick_enabled": { + "name": "idx_provider_endpoints_pick_enabled", + "columns": [ + { + "expression": "vendor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "sort_order", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"provider_endpoints\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_provider_endpoints_created_at": { + "name": "idx_provider_endpoints_created_at", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_provider_endpoints_deleted_at": { + "name": "idx_provider_endpoints_deleted_at", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "provider_endpoints_vendor_id_provider_vendors_id_fk": { + "name": "provider_endpoints_vendor_id_provider_vendors_id_fk", + "tableFrom": "provider_endpoints", + "tableTo": "provider_vendors", + "columnsFrom": [ + "vendor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.provider_groups": { + "name": "provider_groups", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(200)", + "primaryKey": false, + "notNull": true + }, + "cost_multiplier": { + "name": "cost_multiplier", + "type": "numeric(10, 4)", + "primaryKey": false, + "notNull": true, + "default": "'1.0'" + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "provider_groups_name_unique": { + "name": "provider_groups_name_unique", + "nullsNotDistinct": false, + "columns": [ + "name" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.provider_vendors": { + "name": "provider_vendors", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "website_domain": { + "name": "website_domain", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "varchar(200)", + "primaryKey": false, + "notNull": false + }, + "website_url": { + "name": "website_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "favicon_url": { + "name": "favicon_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "uniq_provider_vendors_website_domain": { + "name": "uniq_provider_vendors_website_domain", + "columns": [ + { + "expression": "website_domain", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_provider_vendors_created_at": { + "name": "idx_provider_vendors_created_at", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.providers": { + "name": "providers", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "url": { + "name": "url", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "provider_vendor_id": { + "name": "provider_vendor_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "is_enabled": { + "name": "is_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "weight": { + "name": "weight", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "priority": { + "name": "priority", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "group_priorities": { + "name": "group_priorities", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'null'::jsonb" + }, + "cost_multiplier": { + "name": "cost_multiplier", + "type": "numeric(10, 4)", + "primaryKey": false, + "notNull": false, + "default": "'1.0'" + }, + "group_tag": { + "name": "group_tag", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "provider_type": { + "name": "provider_type", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'claude'" + }, + "preserve_client_ip": { + "name": "preserve_client_ip", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "disable_session_reuse": { + "name": "disable_session_reuse", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "model_redirects": { + "name": "model_redirects", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "allowed_models": { + "name": "allowed_models", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'null'::jsonb" + }, + "allowed_clients": { + "name": "allowed_clients", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "blocked_clients": { + "name": "blocked_clients", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "active_time_start": { + "name": "active_time_start", + "type": "varchar(5)", + "primaryKey": false, + "notNull": false + }, + "active_time_end": { + "name": "active_time_end", + "type": "varchar(5)", + "primaryKey": false, + "notNull": false + }, + "codex_instructions_strategy": { + "name": "codex_instructions_strategy", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false, + "default": "'auto'" + }, + "mcp_passthrough_type": { + "name": "mcp_passthrough_type", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'none'" + }, + "mcp_passthrough_url": { + "name": "mcp_passthrough_url", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false + }, + "limit_5h_usd": { + "name": "limit_5h_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "limit_5h_reset_mode": { + "name": "limit_5h_reset_mode", + "type": "daily_reset_mode", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'rolling'" + }, + "limit_daily_usd": { + "name": "limit_daily_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "daily_reset_mode": { + "name": "daily_reset_mode", + "type": "daily_reset_mode", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'fixed'" + }, + "daily_reset_time": { + "name": "daily_reset_time", + "type": "varchar(5)", + "primaryKey": false, + "notNull": true, + "default": "'00:00'" + }, + "limit_weekly_usd": { + "name": "limit_weekly_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "limit_monthly_usd": { + "name": "limit_monthly_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "limit_total_usd": { + "name": "limit_total_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "total_cost_reset_at": { + "name": "total_cost_reset_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "limit_concurrent_sessions": { + "name": "limit_concurrent_sessions", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "max_retry_attempts": { + "name": "max_retry_attempts", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "circuit_breaker_failure_threshold": { + "name": "circuit_breaker_failure_threshold", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 5 + }, + "circuit_breaker_open_duration": { + "name": "circuit_breaker_open_duration", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 1800000 + }, + "circuit_breaker_half_open_success_threshold": { + "name": "circuit_breaker_half_open_success_threshold", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 2 + }, + "proxy_url": { + "name": "proxy_url", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false + }, + "proxy_fallback_to_direct": { + "name": "proxy_fallback_to_direct", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "custom_headers": { + "name": "custom_headers", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "first_byte_timeout_streaming_ms": { + "name": "first_byte_timeout_streaming_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "streaming_idle_timeout_ms": { + "name": "streaming_idle_timeout_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "request_timeout_non_streaming_ms": { + "name": "request_timeout_non_streaming_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "website_url": { + "name": "website_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "favicon_url": { + "name": "favicon_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cache_ttl_preference": { + "name": "cache_ttl_preference", + "type": "varchar(10)", + "primaryKey": false, + "notNull": false + }, + "swap_cache_ttl_billing": { + "name": "swap_cache_ttl_billing", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "context_1m_preference": { + "name": "context_1m_preference", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "codex_reasoning_effort_preference": { + "name": "codex_reasoning_effort_preference", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "codex_reasoning_summary_preference": { + "name": "codex_reasoning_summary_preference", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "codex_text_verbosity_preference": { + "name": "codex_text_verbosity_preference", + "type": "varchar(10)", + "primaryKey": false, + "notNull": false + }, + "codex_parallel_tool_calls_preference": { + "name": "codex_parallel_tool_calls_preference", + "type": "varchar(10)", + "primaryKey": false, + "notNull": false + }, + "codex_image_generation_preference": { + "name": "codex_image_generation_preference", + "type": "varchar(10)", + "primaryKey": false, + "notNull": false + }, + "codex_service_tier_preference": { + "name": "codex_service_tier_preference", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "anthropic_max_tokens_preference": { + "name": "anthropic_max_tokens_preference", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "anthropic_thinking_budget_preference": { + "name": "anthropic_thinking_budget_preference", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "anthropic_adaptive_thinking": { + "name": "anthropic_adaptive_thinking", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'null'::jsonb" + }, + "gemini_google_search_preference": { + "name": "gemini_google_search_preference", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "tpm": { + "name": "tpm", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "rpm": { + "name": "rpm", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "rpd": { + "name": "rpd", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "cc": { + "name": "cc", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "idx_providers_enabled_priority": { + "name": "idx_providers_enabled_priority", + "columns": [ + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "priority", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "weight", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"providers\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_providers_group": { + "name": "idx_providers_group", + "columns": [ + { + "expression": "group_tag", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"providers\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_providers_vendor_type_url_active": { + "name": "idx_providers_vendor_type_url_active", + "columns": [ + { + "expression": "provider_vendor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "url", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"providers\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_providers_created_at": { + "name": "idx_providers_created_at", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_providers_deleted_at": { + "name": "idx_providers_deleted_at", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_providers_vendor_type": { + "name": "idx_providers_vendor_type", + "columns": [ + { + "expression": "provider_vendor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"providers\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_providers_enabled_vendor_type": { + "name": "idx_providers_enabled_vendor_type", + "columns": [ + { + "expression": "provider_vendor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"providers\".\"deleted_at\" IS NULL AND \"providers\".\"is_enabled\" = true AND \"providers\".\"provider_vendor_id\" IS NOT NULL AND \"providers\".\"provider_vendor_id\" > 0", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "providers_provider_vendor_id_provider_vendors_id_fk": { + "name": "providers_provider_vendor_id_provider_vendors_id_fk", + "tableFrom": "providers", + "tableTo": "provider_vendors", + "columnsFrom": [ + "provider_vendor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.replay_payloads": { + "name": "replay_payloads", + "schema": "", + "columns": { + "replay_id": { + "name": "replay_id", + "type": "varchar(64)", + "primaryKey": true, + "notNull": true + }, + "verifier": { + "name": "verifier", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true + }, + "scope_tag": { + "name": "scope_tag", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true + }, + "key_id": { + "name": "key_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "format": { + "name": "format", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true + }, + "model": { + "name": "model", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false + }, + "status_code": { + "name": "status_code", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "headers_json": { + "name": "headers_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "payload": { + "name": "payload", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "byte_size": { + "name": "byte_size", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "source_message_request_id": { + "name": "source_message_request_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "idx_replay_payloads_key_id": { + "name": "idx_replay_payloads_key_id", + "columns": [ + { + "expression": "key_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_replay_payloads_expires_at": { + "name": "idx_replay_payloads_expires_at", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.request_filters": { + "name": "request_filters", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(100)", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true + }, + "action": { + "name": "action", + "type": "varchar(30)", + "primaryKey": false, + "notNull": true + }, + "match_type": { + "name": "match_type", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "target": { + "name": "target", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "replacement": { + "name": "replacement", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "priority": { + "name": "priority", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "is_enabled": { + "name": "is_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "binding_type": { + "name": "binding_type", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'global'" + }, + "provider_ids": { + "name": "provider_ids", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "group_tags": { + "name": "group_tags", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "rule_mode": { + "name": "rule_mode", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'simple'" + }, + "execution_phase": { + "name": "execution_phase", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'guard'" + }, + "operations": { + "name": "operations", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "idx_request_filters_enabled": { + "name": "idx_request_filters_enabled", + "columns": [ + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "priority", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_request_filters_scope": { + "name": "idx_request_filters_scope", + "columns": [ + { + "expression": "scope", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_request_filters_action": { + "name": "idx_request_filters_action", + "columns": [ + { + "expression": "action", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_request_filters_binding": { + "name": "idx_request_filters_binding", + "columns": [ + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "binding_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_request_filters_phase": { + "name": "idx_request_filters_phase", + "columns": [ + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "execution_phase", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sensitive_words": { + "name": "sensitive_words", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "word": { + "name": "word", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "match_type": { + "name": "match_type", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'contains'" + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_enabled": { + "name": "is_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "idx_sensitive_words_enabled": { + "name": "idx_sensitive_words_enabled", + "columns": [ + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "match_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_sensitive_words_created_at": { + "name": "idx_sensitive_words_created_at", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.system_settings": { + "name": "system_settings", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "site_title": { + "name": "site_title", + "type": "varchar(128)", + "primaryKey": false, + "notNull": true, + "default": "'CC Hub'" + }, + "allow_global_usage_view": { + "name": "allow_global_usage_view", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "currency_display": { + "name": "currency_display", + "type": "varchar(10)", + "primaryKey": false, + "notNull": true, + "default": "'USD'" + }, + "billing_model_source": { + "name": "billing_model_source", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'original'" + }, + "codex_priority_billing_source": { + "name": "codex_priority_billing_source", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'requested'" + }, + "bill_non_successful_requests": { + "name": "bill_non_successful_requests", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "bill_hedge_losers": { + "name": "bill_hedge_losers", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "discovery_enabled": { + "name": "discovery_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "discovery_concurrency": { + "name": "discovery_concurrency", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 2 + }, + "max_discovery_rounds": { + "name": "max_discovery_rounds", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 2 + }, + "discovery_sla_ms": { + "name": "discovery_sla_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 10000 + }, + "sticky_sla_ms": { + "name": "sticky_sla_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 20000 + }, + "racing_total_timeout_ms": { + "name": "racing_total_timeout_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 60000 + }, + "sticky_timeout_cooldown_ms": { + "name": "sticky_timeout_cooldown_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 300000 + }, + "timezone": { + "name": "timezone", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "enable_auto_cleanup": { + "name": "enable_auto_cleanup", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "cleanup_retention_days": { + "name": "cleanup_retention_days", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 30 + }, + "cleanup_schedule": { + "name": "cleanup_schedule", + "type": "varchar(50)", + "primaryKey": false, + "notNull": false, + "default": "'0 2 * * *'" + }, + "cleanup_batch_size": { + "name": "cleanup_batch_size", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 10000 + }, + "enable_client_version_check": { + "name": "enable_client_version_check", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "verbose_provider_error": { + "name": "verbose_provider_error", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "pass_through_upstream_error_message": { + "name": "pass_through_upstream_error_message", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "enable_http2": { + "name": "enable_http2", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "enable_openai_responses_websocket": { + "name": "enable_openai_responses_websocket", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "enable_high_concurrency_mode": { + "name": "enable_high_concurrency_mode", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "intercept_anthropic_warmup_requests": { + "name": "intercept_anthropic_warmup_requests", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "enable_thinking_signature_rectifier": { + "name": "enable_thinking_signature_rectifier", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "enable_thinking_budget_rectifier": { + "name": "enable_thinking_budget_rectifier", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "enable_thinking_effort_conflict_rectifier": { + "name": "enable_thinking_effort_conflict_rectifier", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "enable_gemini_function_id_rectifier": { + "name": "enable_gemini_function_id_rectifier", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "enable_billing_header_rectifier": { + "name": "enable_billing_header_rectifier", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "enable_response_input_rectifier": { + "name": "enable_response_input_rectifier", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "allow_non_conversation_endpoint_provider_fallback": { + "name": "allow_non_conversation_endpoint_provider_fallback", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "fake_streaming_whitelist": { + "name": "fake_streaming_whitelist", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "enable_codex_session_id_completion": { + "name": "enable_codex_session_id_completion", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "enable_claude_metadata_user_id_injection": { + "name": "enable_claude_metadata_user_id_injection", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "enable_response_fixer": { + "name": "enable_response_fixer", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "response_fixer_config": { + "name": "response_fixer_config", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'{\"fixTruncatedJson\":true,\"fixSseFormat\":true,\"fixEncoding\":true,\"maxJsonDepth\":200,\"maxFixSize\":1048576}'::jsonb" + }, + "quota_db_refresh_interval_seconds": { + "name": "quota_db_refresh_interval_seconds", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 10 + }, + "quota_lease_percent_5h": { + "name": "quota_lease_percent_5h", + "type": "numeric(5, 4)", + "primaryKey": false, + "notNull": false, + "default": "'0.05'" + }, + "quota_lease_percent_daily": { + "name": "quota_lease_percent_daily", + "type": "numeric(5, 4)", + "primaryKey": false, + "notNull": false, + "default": "'0.05'" + }, + "quota_lease_percent_weekly": { + "name": "quota_lease_percent_weekly", + "type": "numeric(5, 4)", + "primaryKey": false, + "notNull": false, + "default": "'0.05'" + }, + "quota_lease_percent_monthly": { + "name": "quota_lease_percent_monthly", + "type": "numeric(5, 4)", + "primaryKey": false, + "notNull": false, + "default": "'0.05'" + }, + "quota_lease_cap_usd": { + "name": "quota_lease_cap_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "ip_extraction_config": { + "name": "ip_extraction_config", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "ip_geo_lookup_enabled": { + "name": "ip_geo_lookup_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "public_status_window_hours": { + "name": "public_status_window_hours", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 24 + }, + "public_status_aggregation_interval_minutes": { + "name": "public_status_aggregation_interval_minutes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 5 + }, + "stream_gate_mode": { + "name": "stream_gate_mode", + "type": "varchar(10)", + "primaryKey": false, + "notNull": true, + "default": "'enforce'" + }, + "affinity_ignore_client_session_id": { + "name": "affinity_ignore_client_session_id", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "replay_enabled": { + "name": "replay_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "cache_effectiveness_enabled": { + "name": "cache_effectiveness_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.usage_ledger": { + "name": "usage_ledger", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "request_id": { + "name": "request_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "final_provider_id": { + "name": "final_provider_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "model": { + "name": "model", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false + }, + "original_model": { + "name": "original_model", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false + }, + "actual_response_model": { + "name": "actual_response_model", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false + }, + "endpoint": { + "name": "endpoint", + "type": "varchar(256)", + "primaryKey": false, + "notNull": false + }, + "api_type": { + "name": "api_type", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "session_id": { + "name": "session_id", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "session_identity": { + "name": "session_identity", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "session_identity_kind": { + "name": "session_identity_kind", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "affinity_scope_tag": { + "name": "affinity_scope_tag", + "type": "varchar(16)", + "primaryKey": false, + "notNull": false + }, + "affinity_fingerprint": { + "name": "affinity_fingerprint", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "affinity_fingerprint_chain": { + "name": "affinity_fingerprint_chain", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "is_replay": { + "name": "is_replay", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "replay_source_request_id": { + "name": "replay_source_request_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "status_code": { + "name": "status_code", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "is_success": { + "name": "is_success", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "success_rate_outcome": { + "name": "success_rate_outcome", + "type": "varchar(16)", + "primaryKey": false, + "notNull": false + }, + "blocked_by": { + "name": "blocked_by", + "type": "varchar(50)", + "primaryKey": false, + "notNull": false + }, + "cost_usd": { + "name": "cost_usd", + "type": "numeric(21, 15)", + "primaryKey": false, + "notNull": false, + "default": "'0'" + }, + "cost_multiplier": { + "name": "cost_multiplier", + "type": "numeric(10, 4)", + "primaryKey": false, + "notNull": false + }, + "group_cost_multiplier": { + "name": "group_cost_multiplier", + "type": "numeric(10, 4)", + "primaryKey": false, + "notNull": false + }, + "input_tokens": { + "name": "input_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "output_tokens": { + "name": "output_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "cache_creation_input_tokens": { + "name": "cache_creation_input_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "cache_read_input_tokens": { + "name": "cache_read_input_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "cache_creation_5m_input_tokens": { + "name": "cache_creation_5m_input_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "cache_creation_1h_input_tokens": { + "name": "cache_creation_1h_input_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "cache_ttl_applied": { + "name": "cache_ttl_applied", + "type": "varchar(10)", + "primaryKey": false, + "notNull": false + }, + "context_1m_applied": { + "name": "context_1m_applied", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "swap_cache_ttl_applied": { + "name": "swap_cache_ttl_applied", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "ttfb_ms": { + "name": "ttfb_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "first_byte_ms": { + "name": "first_byte_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "client_ip": { + "name": "client_ip", + "type": "varchar(45)", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "idx_usage_ledger_request_id": { + "name": "idx_usage_ledger_request_id", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_user_created_at": { + "name": "idx_usage_ledger_user_created_at", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_ledger\".\"blocked_by\" IS NULL AND \"usage_ledger\".\"is_replay\" = false", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_user_id_reset": { + "name": "idx_usage_ledger_user_id_reset", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_key_created_at": { + "name": "idx_usage_ledger_key_created_at", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_ledger\".\"blocked_by\" IS NULL AND \"usage_ledger\".\"is_replay\" = false", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_provider_created_at": { + "name": "idx_usage_ledger_provider_created_at", + "columns": [ + { + "expression": "final_provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_ledger\".\"blocked_by\" IS NULL AND \"usage_ledger\".\"is_replay\" = false", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_created_at_minute": { + "name": "idx_usage_ledger_created_at_minute", + "columns": [ + { + "expression": "date_trunc('minute', \"created_at\" AT TIME ZONE 'UTC')", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_created_at_desc_id": { + "name": "idx_usage_ledger_created_at_desc_id", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_session_id": { + "name": "idx_usage_ledger_session_id", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_ledger\".\"session_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_session_identity_created_at": { + "name": "idx_usage_ledger_session_identity_created_at", + "columns": [ + { + "expression": "COALESCE(\"session_identity\", \"session_id\")", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "\"created_at\" DESC NULLS LAST", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_ledger\".\"blocked_by\" IS NULL AND \"usage_ledger\".\"is_replay\" = false", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_session_identity": { + "name": "idx_usage_ledger_session_identity", + "columns": [ + { + "expression": "COALESCE(\"session_identity\", \"session_id\")", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_model": { + "name": "idx_usage_ledger_model", + "columns": [ + { + "expression": "model", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_ledger\".\"model\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_key_cost": { + "name": "idx_usage_ledger_key_cost", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "cost_usd", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "endpoint", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_ledger\".\"blocked_by\" IS NULL AND \"usage_ledger\".\"is_replay\" = false", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_user_cost_cover": { + "name": "idx_usage_ledger_user_cost_cover", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "cost_usd", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "endpoint", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_ledger\".\"blocked_by\" IS NULL AND \"usage_ledger\".\"is_replay\" = false", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_provider_cost_cover": { + "name": "idx_usage_ledger_provider_cost_cover", + "columns": [ + { + "expression": "final_provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "cost_usd", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "endpoint", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_ledger\".\"blocked_by\" IS NULL AND \"usage_ledger\".\"is_replay\" = false", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_key_created_at_desc_cover": { + "name": "idx_usage_ledger_key_created_at_desc_cover", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "\"created_at\" DESC NULLS LAST", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "final_provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_ledger\".\"blocked_by\" IS NULL AND \"usage_ledger\".\"is_replay\" = false", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.users": { + "name": "users", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "role": { + "name": "role", + "type": "varchar", + "primaryKey": false, + "notNull": false, + "default": "'user'" + }, + "rpm_limit": { + "name": "rpm_limit", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "daily_limit_usd": { + "name": "daily_limit_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "provider_group": { + "name": "provider_group", + "type": "varchar(200)", + "primaryKey": false, + "notNull": false, + "default": "'default'" + }, + "tags": { + "name": "tags", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'[]'::jsonb" + }, + "limit_5h_usd": { + "name": "limit_5h_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "limit_5h_reset_mode": { + "name": "limit_5h_reset_mode", + "type": "daily_reset_mode", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'rolling'" + }, + "limit_weekly_usd": { + "name": "limit_weekly_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "limit_monthly_usd": { + "name": "limit_monthly_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "limit_total_usd": { + "name": "limit_total_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "cost_reset_at": { + "name": "cost_reset_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "limit_5h_cost_reset_at": { + "name": "limit_5h_cost_reset_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "limit_concurrent_sessions": { + "name": "limit_concurrent_sessions", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "daily_reset_mode": { + "name": "daily_reset_mode", + "type": "daily_reset_mode", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'fixed'" + }, + "daily_reset_time": { + "name": "daily_reset_time", + "type": "varchar(5)", + "primaryKey": false, + "notNull": true, + "default": "'00:00'" + }, + "is_enabled": { + "name": "is_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "allowed_clients": { + "name": "allowed_clients", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'[]'::jsonb" + }, + "allowed_models": { + "name": "allowed_models", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'[]'::jsonb" + }, + "blocked_clients": { + "name": "blocked_clients", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "idx_users_active_role_sort": { + "name": "idx_users_active_role_sort", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "role", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"users\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_users_enabled_expires_at": { + "name": "idx_users_enabled_expires_at", + "columns": [ + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"users\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_users_tags_gin": { + "name": "idx_users_tags_gin", + "columns": [ + { + "expression": "tags", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"users\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "gin", + "with": {} + }, + "idx_users_created_at": { + "name": "idx_users_created_at", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_users_deleted_at": { + "name": "idx_users_deleted_at", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.webhook_targets": { + "name": "webhook_targets", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(100)", + "primaryKey": false, + "notNull": true + }, + "provider_type": { + "name": "provider_type", + "type": "webhook_provider_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "webhook_url": { + "name": "webhook_url", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": false + }, + "telegram_bot_token": { + "name": "telegram_bot_token", + "type": "varchar(256)", + "primaryKey": false, + "notNull": false + }, + "telegram_chat_id": { + "name": "telegram_chat_id", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "dingtalk_secret": { + "name": "dingtalk_secret", + "type": "varchar(256)", + "primaryKey": false, + "notNull": false + }, + "custom_template": { + "name": "custom_template", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "custom_headers": { + "name": "custom_headers", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "proxy_url": { + "name": "proxy_url", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false + }, + "proxy_fallback_to_direct": { + "name": "proxy_fallback_to_direct", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "is_enabled": { + "name": "is_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "last_test_at": { + "name": "last_test_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_test_result": { + "name": "last_test_result", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.daily_reset_mode": { + "name": "daily_reset_mode", + "schema": "public", + "values": [ + "fixed", + "rolling" + ] + }, + "public.notification_type": { + "name": "notification_type", + "schema": "public", + "values": [ + "circuit_breaker", + "daily_leaderboard", + "cost_alert", + "cache_hit_rate_alert" + ] + }, + "public.webhook_provider_type": { + "name": "webhook_provider_type", + "schema": "public", + "values": [ + "wechat", + "feishu", + "dingtalk", + "telegram", + "custom" + ] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index 025658d86..f09e1f133 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -827,6 +827,13 @@ "when": 1785635169798, "tag": "0117_loving_whiplash", "breakpoints": true + }, + { + "idx": 118, + "version": "7", + "when": 1785688550789, + "tag": "0118_bright_sunspot", + "breakpoints": true } ] } \ No newline at end of file diff --git a/src/drizzle/schema.ts b/src/drizzle/schema.ts index ebb4d600a..f0684ff8e 100644 --- a/src/drizzle/schema.ts +++ b/src/drizzle/schema.ts @@ -661,6 +661,16 @@ export const messageRequest = pgTable('message_request', { ) .on(table.providerId, table.createdAt.desc()) .where(sql`${table.deletedAt} IS NULL AND ${table.statusCode} IS NOT NULL`), + messageRequestProxyStatusActiveIdx: index('idx_message_request_proxy_status_active') + .on(sql`${table.createdAt} DESC NULLS LAST`, table.userId) + .where( + sql`${table.deletedAt} IS NULL AND ${table.isReplay} = false AND ${table.statusCode} IS NULL AND (${table.blockedBy} IS NULL OR ${table.blockedBy} <> 'warmup')` + ), + messageRequestProxyStatusLatestIdx: index('idx_message_request_proxy_status_latest') + .on(table.userId, sql`${table.updatedAt} DESC NULLS LAST`, table.id.desc()) + .where( + sql`${table.deletedAt} IS NULL AND ${table.isReplay} = false AND ${table.statusCode} IS NOT NULL AND (${table.blockedBy} IS NULL OR ${table.blockedBy} <> 'warmup')` + ), // Session 查询索引（按 session 聚合查看对话） messageRequestSessionIdIdx: index('idx_message_request_session_id').on(table.sessionId).where(sql`${table.deletedAt} IS NULL`), // Session ID 前缀查询索引（LIKE 'prefix%'，可稳定命中 B-tree） @@ -670,7 +680,11 @@ export const messageRequest = pgTable('message_request', { messageRequestSessionIdentityCreatedAtIdx: index( 'idx_message_request_session_identity_created_at' ) - .on(sql`COALESCE(${table.sessionIdentity}, ${table.sessionId})`, table.createdAt.desc()) + .on( + sql`COALESCE(${table.sessionIdentity}, ${table.sessionId})`, + sql`${table.createdAt} DESC NULLS LAST`, + table.id.desc() + ) .where(sql`${table.deletedAt} IS NULL`), // Endpoint 过滤查询索引（仅针对未删除数据） messageRequestEndpointIdx: index('idx_message_request_endpoint').on(table.endpoint).where(sql`${table.deletedAt} IS NULL`), @@ -1211,6 +1225,7 @@ export const usageLedger = pgTable('usage_ledger', { usageLedgerUserCreatedAtIdx: index('idx_usage_ledger_user_created_at') .on(table.userId, table.createdAt) .where(sql`${table.blockedBy} IS NULL AND ${table.isReplay} = false`), + usageLedgerUserIdResetIdx: index('idx_usage_ledger_user_id_reset').on(table.userId), usageLedgerKeyCreatedAtIdx: index('idx_usage_ledger_key_created_at') .on(table.key, table.createdAt) .where(sql`${table.blockedBy} IS NULL AND ${table.isReplay} = false`), @@ -1228,7 +1243,11 @@ export const usageLedger = pgTable('usage_ledger', { usageLedgerSessionIdentityCreatedAtIdx: index( 'idx_usage_ledger_session_identity_created_at' ) - .on(sql`COALESCE(${table.sessionIdentity}, ${table.sessionId})`, table.createdAt.desc()) + .on( + sql`COALESCE(${table.sessionIdentity}, ${table.sessionId})`, + table.userId, + sql`${table.createdAt} DESC NULLS LAST` + ) .where(sql`${table.blockedBy} IS NULL AND ${table.isReplay} = false`), usageLedgerSessionIdentityIdx: index('idx_usage_ledger_session_identity') .on(sql`COALESCE(${table.sessionIdentity}, ${table.sessionId})`), diff --git a/src/lib/migrate.ts b/src/lib/migrate.ts index 722cad64c..b2e44bcf7 100644 --- a/src/lib/migrate.ts +++ b/src/lib/migrate.ts @@ -214,6 +214,7 @@ export async function runMigrations() { logger.info("Waiting for database migration lock..."); await migrationClient`SELECT pg_advisory_lock(hashtext(${MIGRATION_ADVISORY_LOCK_NAME}))`; logger.info("Database migration lock acquired"); + await migrationClient`SET search_path TO public`; // 获取迁移文件路径 const migrationsFolder = path.join(process.cwd(), "drizzle"); diff --git a/src/lib/migrations/session-replay-index-preflight.ts b/src/lib/migrations/session-replay-index-preflight.ts index 618487bb6..97de7ba2a 100644 --- a/src/lib/migrations/session-replay-index-preflight.ts +++ b/src/lib/migrations/session-replay-index-preflight.ts @@ -1,6 +1,8 @@ export const SESSION_REPLAY_MIGRATION_CREATED_AT = 1785563419224; export const SESSION_IDENTITY_INDEX_MIGRATION_CREATED_AT = 1785635169798; +export const DATABASE_TIMEOUT_INDEX_MIGRATION_CREATED_AT = 1785688550789; export const SESSION_REPLAY_INDEX_MARKER = "cch:migration:0116:session-replay-index:v1"; +export const DATABASE_TIMEOUT_INDEX_MARKER = "cch:migration:0118:database-timeout-index:v2"; export type MigrationIndexState = { exists: boolean; @@ -16,73 +18,104 @@ export type MigrationIndexPreflightExecutor = { export type SessionReplayIndexSpec = { canonicalName: string; temporaryName: string; + marker: string; definition: string; }; export const SESSION_REPLAY_INDEX_SPECS: readonly SessionReplayIndexSpec[] = [ { canonicalName: "idx_message_request_session_identity_created_at", - temporaryName: "cch_0116_tmp_01", + temporaryName: "cch_0118_tmp_01", + marker: DATABASE_TIMEOUT_INDEX_MARKER, definition: - 'ON "message_request" USING btree (COALESCE("session_identity", "session_id"),"created_at" DESC NULLS LAST) WHERE "message_request"."deleted_at" IS NULL', + 'ON "public"."message_request" USING btree (COALESCE("session_identity", "session_id"),"created_at" DESC NULLS LAST,"id" DESC NULLS LAST) WHERE "message_request"."deleted_at" IS NULL', }, { canonicalName: "idx_usage_ledger_session_identity_created_at", - temporaryName: "cch_0116_tmp_02", + temporaryName: "cch_0118_tmp_02", + marker: DATABASE_TIMEOUT_INDEX_MARKER, definition: - 'ON "usage_ledger" USING btree (COALESCE("session_identity", "session_id"),"created_at" DESC NULLS LAST) WHERE "usage_ledger"."blocked_by" IS NULL AND "usage_ledger"."is_replay" = false', + 'ON "public"."usage_ledger" USING btree (COALESCE("session_identity", "session_id"),"user_id","created_at" DESC NULLS LAST) WHERE "usage_ledger"."blocked_by" IS NULL AND "usage_ledger"."is_replay" = false', + }, + { + canonicalName: "idx_message_request_proxy_status_active", + temporaryName: "cch_0118_tmp_03", + marker: DATABASE_TIMEOUT_INDEX_MARKER, + definition: + 'ON "public"."message_request" USING btree ("created_at" DESC NULLS LAST,"user_id") WHERE "message_request"."deleted_at" IS NULL AND "message_request"."is_replay" = false AND "message_request"."status_code" IS NULL AND ("message_request"."blocked_by" IS NULL OR "message_request"."blocked_by" <> \'warmup\')', + }, + { + canonicalName: "idx_message_request_proxy_status_latest", + temporaryName: "cch_0118_tmp_04", + marker: DATABASE_TIMEOUT_INDEX_MARKER, + definition: + 'ON "public"."message_request" USING btree ("user_id","updated_at" DESC NULLS LAST,"id" DESC NULLS LAST) WHERE "message_request"."deleted_at" IS NULL AND "message_request"."is_replay" = false AND "message_request"."status_code" IS NOT NULL AND ("message_request"."blocked_by" IS NULL OR "message_request"."blocked_by" <> \'warmup\')', + }, + { + canonicalName: "idx_usage_ledger_user_id_reset", + temporaryName: "cch_0118_tmp_05", + marker: DATABASE_TIMEOUT_INDEX_MARKER, + definition: 'ON "public"."usage_ledger" USING btree ("user_id")', }, { canonicalName: "idx_usage_ledger_session_identity", temporaryName: "cch_0117_tmp_01", + marker: SESSION_REPLAY_INDEX_MARKER, definition: 'ON "usage_ledger" USING btree (COALESCE("session_identity", "session_id"))', }, { canonicalName: "idx_usage_ledger_user_created_at", temporaryName: "cch_0116_tmp_03", + marker: SESSION_REPLAY_INDEX_MARKER, definition: 'ON "usage_ledger" USING btree ("user_id","created_at") WHERE "usage_ledger"."blocked_by" IS NULL AND "usage_ledger"."is_replay" = false', }, { canonicalName: "idx_usage_ledger_key_created_at", temporaryName: "cch_0116_tmp_04", + marker: SESSION_REPLAY_INDEX_MARKER, definition: 'ON "usage_ledger" USING btree ("key","created_at") WHERE "usage_ledger"."blocked_by" IS NULL AND "usage_ledger"."is_replay" = false', }, { canonicalName: "idx_usage_ledger_provider_created_at", temporaryName: "cch_0116_tmp_05", + marker: SESSION_REPLAY_INDEX_MARKER, definition: 'ON "usage_ledger" USING btree ("final_provider_id","created_at") WHERE "usage_ledger"."blocked_by" IS NULL AND "usage_ledger"."is_replay" = false', }, { canonicalName: "idx_usage_ledger_key_cost", temporaryName: "cch_0116_tmp_06", + marker: SESSION_REPLAY_INDEX_MARKER, definition: 'ON "usage_ledger" USING btree ("key","created_at","cost_usd","endpoint") WHERE "usage_ledger"."blocked_by" IS NULL AND "usage_ledger"."is_replay" = false', }, { canonicalName: "idx_usage_ledger_user_cost_cover", temporaryName: "cch_0116_tmp_07", + marker: SESSION_REPLAY_INDEX_MARKER, definition: 'ON "usage_ledger" USING btree ("user_id","created_at","cost_usd","endpoint") WHERE "usage_ledger"."blocked_by" IS NULL AND "usage_ledger"."is_replay" = false', }, { canonicalName: "idx_usage_ledger_provider_cost_cover", temporaryName: "cch_0116_tmp_08", + marker: SESSION_REPLAY_INDEX_MARKER, definition: 'ON "usage_ledger" USING btree ("final_provider_id","created_at","cost_usd","endpoint") WHERE "usage_ledger"."blocked_by" IS NULL AND "usage_ledger"."is_replay" = false', }, { canonicalName: "idx_usage_ledger_key_created_at_desc_cover", temporaryName: "cch_0116_tmp_09", + marker: SESSION_REPLAY_INDEX_MARKER, definition: 'ON "usage_ledger" USING btree ("key","created_at" DESC NULLS LAST,"final_provider_id") WHERE "usage_ledger"."blocked_by" IS NULL AND "usage_ledger"."is_replay" = false', }, ]; -function isValidated0116Index(state: MigrationIndexState): boolean { - return state.exists && state.valid && state.marker === SESSION_REPLAY_INDEX_MARKER; +function isValidatedIndex(state: MigrationIndexState, marker: string): boolean { + return state.exists && state.valid && state.marker === marker; } async function ensurePreflightColumns(executor: MigrationIndexPreflightExecutor): Promise { @@ -90,6 +123,8 @@ async function ensurePreflightColumns(executor: MigrationIndexPreflightExecutor) try { await executor.execute(`ALTER TABLE "message_request" ADD COLUMN IF NOT EXISTS "session_identity" varchar(64); +ALTER TABLE "message_request" + ADD COLUMN IF NOT EXISTS "is_replay" boolean DEFAULT false NOT NULL; ALTER TABLE "usage_ledger" ADD COLUMN IF NOT EXISTS "session_identity" varchar(64); ALTER TABLE "usage_ledger" @@ -110,7 +145,7 @@ export async function runSessionReplayIndexPreflight( for (const spec of specs) { const canonical = await executor.inspectIndex(spec.canonicalName); - if (isValidated0116Index(canonical)) { + if (isValidatedIndex(canonical, spec.marker)) { const staleTemp = await executor.inspectIndex(spec.temporaryName); if (staleTemp.exists) { await executor.execute(`DROP INDEX CONCURRENTLY IF EXISTS "${spec.temporaryName}"`); @@ -119,7 +154,7 @@ export async function runSessionReplayIndexPreflight( } let temporary = await executor.inspectIndex(spec.temporaryName); - if (!isValidated0116Index(temporary)) { + if (!isValidatedIndex(temporary, spec.marker)) { if (temporary.exists) { await executor.execute(`DROP INDEX CONCURRENTLY IF EXISTS "${spec.temporaryName}"`); } @@ -127,22 +162,24 @@ export async function runSessionReplayIndexPreflight( `CREATE INDEX CONCURRENTLY "${spec.temporaryName}" ${spec.definition}` ); await executor.execute( - `COMMENT ON INDEX "${spec.temporaryName}" IS '${SESSION_REPLAY_INDEX_MARKER}'` + `COMMENT ON INDEX "public"."${spec.temporaryName}" IS '${spec.marker}'` ); temporary = await executor.inspectIndex(spec.temporaryName); - if (!isValidated0116Index(temporary)) { - throw new Error(`0116 preflight produced an invalid index: ${spec.temporaryName}`); + if (!isValidatedIndex(temporary, spec.marker)) { + throw new Error(`Concurrent preflight produced an invalid index: ${spec.temporaryName}`); } } if (canonical.exists) { - await executor.execute(`DROP INDEX CONCURRENTLY IF EXISTS "${spec.canonicalName}"`); + await executor.execute(`DROP INDEX CONCURRENTLY IF EXISTS "public"."${spec.canonicalName}"`); } - await executor.execute(`ALTER INDEX "${spec.temporaryName}" RENAME TO "${spec.canonicalName}"`); + await executor.execute( + `ALTER INDEX "public"."${spec.temporaryName}" RENAME TO "${spec.canonicalName}"` + ); const replaced = await executor.inspectIndex(spec.canonicalName); - if (!isValidated0116Index(replaced)) { - throw new Error(`0116 preflight failed to install index: ${spec.canonicalName}`); + if (!isValidatedIndex(replaced, spec.marker)) { + throw new Error(`Concurrent preflight failed to install index: ${spec.canonicalName}`); } } } @@ -159,7 +196,7 @@ export async function runSessionReplayMigrationPlan(input: { baseTablesReady && (latestMigrationCreatedAt == null || !Number.isFinite(latestMigrationCreatedAt) || - latestMigrationCreatedAt < SESSION_IDENTITY_INDEX_MIGRATION_CREATED_AT) + latestMigrationCreatedAt < DATABASE_TIMEOUT_INDEX_MIGRATION_CREATED_AT) ) { await runIndexPreflight({ ensureColumns: true }); } diff --git a/tests/unit/drizzle/database-timeout-migration.test.ts b/tests/unit/drizzle/database-timeout-migration.test.ts new file mode 100644 index 000000000..1d9716e42 --- /dev/null +++ b/tests/unit/drizzle/database-timeout-migration.test.ts @@ -0,0 +1,41 @@ +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { describe, expect, test } from "vitest"; + +const migration = readFileSync( + resolve(process.cwd(), "drizzle/0118_bright_sunspot.sql"), + "utf-8" +); + +describe("0118 database timeout indexes", () => { + test.each([ + "idx_message_request_session_identity_created_at", + "idx_usage_ledger_session_identity_created_at", + "idx_message_request_proxy_status_active", + "idx_message_request_proxy_status_latest", + "idx_usage_ledger_user_id_reset", + ])("keeps %s idempotent for fresh databases", (indexName) => { + expect(migration).toContain(`CREATE INDEX IF NOT EXISTS "${indexName}"`); + }); + + test("does not drop or rebuild large indexes transactionally", () => { + expect(migration).not.toMatch(/DROP\s+INDEX/i); + expect(migration).not.toMatch(/CREATE\s+INDEX\s+(?!IF\s+NOT\s+EXISTS)/i); + }); + + test("records the approved Session and Proxy Status index shapes", () => { + expect(migration).toContain( + 'COALESCE("session_identity", "session_id"),"created_at" DESC NULLS LAST,"id" DESC NULLS LAST' + ); + expect(migration).toContain( + 'COALESCE("session_identity", "session_id"),"user_id","created_at" DESC NULLS LAST' + ); + expect(migration).toContain('"created_at" DESC NULLS LAST,"user_id"'); + expect(migration).toContain( + '"user_id","updated_at" DESC NULLS LAST,"id" DESC NULLS LAST' + ); + expect(migration).toContain( + 'CREATE INDEX IF NOT EXISTS "idx_usage_ledger_user_id_reset" ON "usage_ledger" USING btree ("user_id")' + ); + }); +}); diff --git a/tests/unit/drizzle/proxy-status-indexes.test.ts b/tests/unit/drizzle/proxy-status-indexes.test.ts new file mode 100644 index 000000000..4fb75d21e --- /dev/null +++ b/tests/unit/drizzle/proxy-status-indexes.test.ts @@ -0,0 +1,37 @@ +import type { SQL } from "drizzle-orm"; +import { getTableConfig, PgDialect } from "drizzle-orm/pg-core"; +import { describe, expect, it } from "vitest"; +import { messageRequest } from "@/drizzle/schema"; + +const dialect = new PgDialect(); +const compile = (value: unknown) => dialect.sqlToQuery(value as SQL).sql.toLowerCase(); + +describe("Proxy Status indexes", () => { + it("indexes bounded active requests", () => { + const index = getTableConfig(messageRequest).indexes.find( + (entry) => entry.config.name === "idx_message_request_proxy_status_active" + ); + expect(index).toBeDefined(); + expect(compile(index?.config.columns[0])).toContain("created_at"); + expect(compile(index?.config.columns[0])).toContain("desc nulls last"); + expect(index?.config.columns[1]).toMatchObject({ name: "user_id" }); + const predicate = compile(index?.config.where); + expect(predicate).toContain('"status_code" is null'); + expect(predicate).toContain('"is_replay" = false'); + expect(predicate).toContain("warmup"); + }); + + it("indexes the deterministic latest finalized request per user", () => { + const index = getTableConfig(messageRequest).indexes.find( + (entry) => entry.config.name === "idx_message_request_proxy_status_latest" + ); + expect(index).toBeDefined(); + expect(index?.config.columns[0]).toMatchObject({ name: "user_id" }); + expect(compile(index?.config.columns[1])).toContain("updated_at"); + expect(compile(index?.config.columns[1])).toContain("desc nulls last"); + expect(index?.config.columns[2]).toMatchObject({ name: "id", indexConfig: { order: "desc" } }); + const predicate = compile(index?.config.where); + expect(predicate).toContain('"status_code" is not null'); + expect(predicate).toContain('"is_replay" = false'); + }); +}); diff --git a/tests/unit/drizzle/session-identity-indexes.test.ts b/tests/unit/drizzle/session-identity-indexes.test.ts index 5d5841191..e3719d872 100644 --- a/tests/unit/drizzle/session-identity-indexes.test.ts +++ b/tests/unit/drizzle/session-identity-indexes.test.ts @@ -34,6 +34,33 @@ describe("Session identity query indexes", () => { expect(compileSql(index?.config.where as SQL)).toContain(predicate); }); + test("message_request identity index supports deterministic latest-row lookup", () => { + const index = getTableConfig(messageRequest).indexes.find( + (entry) => entry.config.name === "idx_message_request_session_identity_created_at" + ); + expect(index).toBeDefined(); + expect(index?.config.columns).toHaveLength(3); + const createdAt = compileSql(index?.config.columns[1] as SQL); + expect(createdAt).toContain("created_at"); + expect(createdAt).toContain("desc nulls last"); + expect(index?.config.columns[2]).toMatchObject({ + name: "id", + indexConfig: { order: "desc" }, + }); + }); + + test("usage_ledger identity index includes owner before creation time", () => { + const index = getTableConfig(usageLedger).indexes.find( + (entry) => entry.config.name === "idx_usage_ledger_session_identity_created_at" + ); + expect(index).toBeDefined(); + expect(index?.config.columns).toHaveLength(3); + expect(index?.config.columns[1]).toMatchObject({ name: "user_id" }); + const createdAt = compileSql(index?.config.columns[2] as SQL); + expect(createdAt).toContain("created_at"); + expect(createdAt).toContain("desc nulls last"); + }); + test("usage_ledger has an unfiltered identity index for grouped source-ID hydration", () => { const index = getTableConfig(usageLedger).indexes.find( (entry) => entry.config.name === "idx_usage_ledger_session_identity" diff --git a/tests/unit/lib/session-replay-index-preflight.test.ts b/tests/unit/lib/session-replay-index-preflight.test.ts index 9e24e02be..d71eadaca 100644 --- a/tests/unit/lib/session-replay-index-preflight.test.ts +++ b/tests/unit/lib/session-replay-index-preflight.test.ts @@ -1,5 +1,7 @@ import { describe, expect, test, vi } from "vitest"; import { + DATABASE_TIMEOUT_INDEX_MARKER, + DATABASE_TIMEOUT_INDEX_MIGRATION_CREATED_AT, SESSION_REPLAY_INDEX_MARKER, SESSION_REPLAY_INDEX_SPECS, SESSION_REPLAY_MIGRATION_CREATED_AT, @@ -17,21 +19,22 @@ function createFakeExecutor(initial: Record = {}) { return; } - const commentName = sql.match(/^COMMENT ON INDEX "([^"]+)"/)?.[1]; - if (commentName) { + const comment = sql.match(/^COMMENT ON INDEX (?:"public"\.)?"([^"]+)" IS '([^']+)'/)?.slice(1); + if (comment) { + const [commentName, marker] = comment; const state = states.get(commentName); if (!state) throw new Error(`missing index ${commentName}`); - states.set(commentName, { ...state, marker: SESSION_REPLAY_INDEX_MARKER }); + states.set(commentName, { ...state, marker }); return; } - const dropName = sql.match(/^DROP INDEX CONCURRENTLY IF EXISTS "([^"]+)"/)?.[1]; + const dropName = sql.match(/^DROP INDEX CONCURRENTLY IF EXISTS (?:"public"\.)?"([^"]+)"/)?.[1]; if (dropName) { states.delete(dropName); return; } - const rename = sql.match(/^ALTER INDEX "([^"]+)" RENAME TO "([^"]+)"/)?.slice(1); + const rename = sql.match(/^ALTER INDEX (?:"public"\.)?"([^"]+)" RENAME TO "([^"]+)"/)?.slice(1); if (rename) { const [from, to] = rename; const state = states.get(from); @@ -46,7 +49,7 @@ function createFakeExecutor(initial: Record = {}) { return { executor: { execute, inspectIndex }, execute, inspectIndex, states }; } -describe("0116 concurrent index preflight", () => { +describe("database index concurrent preflight", () => { const spec = SESSION_REPLAY_INDEX_SPECS[0]; const hydrationSpec = SESSION_REPLAY_INDEX_SPECS.find( (candidate) => candidate.canonicalName === "idx_usage_ledger_session_identity" @@ -76,7 +79,7 @@ describe("0116 concurrent index preflight", () => { expect(states.get(spec.canonicalName)).toEqual({ exists: true, valid: true, - marker: SESSION_REPLAY_INDEX_MARKER, + marker: spec.marker, }); expect(states.has(spec.temporaryName)).toBe(false); @@ -85,10 +88,10 @@ describe("0116 concurrent index preflight", () => { statement.startsWith("CREATE INDEX CONCURRENTLY") ); const dropAt = sql.findIndex((statement) => - statement.includes(`DROP INDEX CONCURRENTLY IF EXISTS "${spec.canonicalName}"`) + statement.includes(`DROP INDEX CONCURRENTLY IF EXISTS "public"."${spec.canonicalName}"`) ); const renameAt = sql.findIndex((statement) => - statement.includes(`ALTER INDEX "${spec.temporaryName}" RENAME TO`) + statement.includes(`ALTER INDEX "public"."${spec.temporaryName}" RENAME TO`) ); expect(createAt).toBeGreaterThanOrEqual(0); expect(dropAt).toBeGreaterThan(createAt); @@ -100,13 +103,13 @@ describe("0116 concurrent index preflight", () => { [spec.temporaryName]: { exists: true, valid: true, - marker: SESSION_REPLAY_INDEX_MARKER, + marker: spec.marker, }, }); await runSessionReplayIndexPreflight(executor, [spec]); - expect(states.get(spec.canonicalName)?.marker).toBe(SESSION_REPLAY_INDEX_MARKER); + expect(states.get(spec.canonicalName)?.marker).toBe(spec.marker); expect(execute.mock.calls.flat().some((sql) => sql.startsWith("CREATE INDEX"))).toBe(false); }); @@ -141,14 +144,14 @@ describe("0116 concurrent index preflight", () => { [spec.canonicalName]: { exists: true, valid: true, - marker: SESSION_REPLAY_INDEX_MARKER, + marker: spec.marker, }, [spec.temporaryName]: { exists: true, valid: false, marker: null }, }); await runSessionReplayIndexPreflight(executor, [spec]); - expect(states.get(spec.canonicalName)?.marker).toBe(SESSION_REPLAY_INDEX_MARKER); + expect(states.get(spec.canonicalName)?.marker).toBe(spec.marker); expect(states.has(spec.temporaryName)).toBe(false); expect( execute.mock.calls @@ -162,7 +165,7 @@ describe("0116 concurrent index preflight", () => { [spec.canonicalName]: { exists: true, valid: true, - marker: SESSION_REPLAY_INDEX_MARKER, + marker: spec.marker, }, }); @@ -172,9 +175,71 @@ describe("0116 concurrent index preflight", () => { expect(statements.some((statement) => statement.includes("ALTER TABLE"))).toBe(false); expect(statements.some((statement) => statement.includes("lock_timeout"))).toBe(false); }); + + test("adds pre-0116 Replay columns before building timeout indexes", async () => { + const { executor, execute } = createFakeExecutor(); + + await runSessionReplayIndexPreflight(executor, [spec]); + + const statements = execute.mock.calls.map(([statement]) => statement); + const ensureColumns = statements.find((statement) => statement.includes("ALTER TABLE")); + const createIndexAt = statements.findIndex((statement) => + statement.startsWith("CREATE INDEX CONCURRENTLY") + ); + const ensureColumnsAt = ensureColumns ? statements.indexOf(ensureColumns) : -1; + + expect(ensureColumns).toContain('ALTER TABLE "message_request"'); + expect(ensureColumns).toContain( + 'ADD COLUMN IF NOT EXISTS "is_replay" boolean DEFAULT false NOT NULL' + ); + expect(ensureColumnsAt).toBeGreaterThanOrEqual(0); + expect(createIndexAt).toBeGreaterThan(ensureColumnsAt); + }); + + test("keeps validated v1 indexes while upgrading only the v2 timeout indexes", async () => { + const legacySpec = SESSION_REPLAY_INDEX_SPECS.find( + (candidate) => candidate.marker === SESSION_REPLAY_INDEX_MARKER + ); + if (!legacySpec) throw new Error("missing v1 index spec"); + const { executor, execute } = createFakeExecutor({ + [legacySpec.canonicalName]: { + exists: true, + valid: true, + marker: SESSION_REPLAY_INDEX_MARKER, + }, + [spec.canonicalName]: { + exists: true, + valid: true, + marker: SESSION_REPLAY_INDEX_MARKER, + }, + }); + + await runSessionReplayIndexPreflight(executor, [legacySpec, spec]); + + const createStatements = execute.mock.calls + .map(([statement]) => statement) + .filter((statement) => statement.startsWith("CREATE INDEX CONCURRENTLY")); + expect(createStatements).toHaveLength(1); + expect(createStatements[0]).toContain(spec.temporaryName); + expect(createStatements[0]).not.toContain(legacySpec.temporaryName); + }); + + test("uses the v2 marker and public-qualified definitions for timeout indexes", () => { + const timeoutSpecs = SESSION_REPLAY_INDEX_SPECS.filter( + (candidate) => candidate.marker === DATABASE_TIMEOUT_INDEX_MARKER + ); + + expect(timeoutSpecs).toHaveLength(5); + expect( + timeoutSpecs.every((candidate) => candidate.temporaryName.startsWith("cch_0118_tmp_")) + ).toBe(true); + expect(timeoutSpecs.every((candidate) => candidate.definition.includes('ON "public".'))).toBe( + true + ); + }); }); -describe("0116 migration orchestration", () => { +describe("database index migration orchestration", () => { test("preflights the unfiltered ledger identity index before migration 0117", async () => { const events: string[] = []; await runSessionReplayMigrationPlan({ @@ -237,6 +302,32 @@ describe("0116 migration orchestration", () => { expect(calls).toEqual(["indexes", "migrate", "indexes"]); }); + test("prebuilds timeout indexes before upgrading an existing 0117 database", async () => { + const calls: string[] = []; + + await runSessionReplayMigrationPlan({ + baseTablesReady: true, + latestMigrationCreatedAt: DATABASE_TIMEOUT_INDEX_MIGRATION_CREATED_AT - 1, + migrate: async () => calls.push("migrate"), + runIndexPreflight: async () => calls.push("indexes"), + }); + + expect(calls).toEqual(["indexes", "migrate", "indexes"]); + }); + + test("runs only postflight after migration 0118 is already recorded", async () => { + const calls: string[] = []; + + await runSessionReplayMigrationPlan({ + baseTablesReady: true, + latestMigrationCreatedAt: DATABASE_TIMEOUT_INDEX_MIGRATION_CREATED_AT, + migrate: async () => calls.push("migrate"), + runIndexPreflight: async () => calls.push("indexes"), + }); + + expect(calls).toEqual(["migrate", "indexes"]); + }); + test("fails the migration flow when concurrent index postflight fails", async () => { await expect( runSessionReplayMigrationPlan({ From af545d84c50ff2fa9422bd66553eda2487492a14 Mon Sep 17 00:00:00 2001 From: ding113 Date: Mon, 3 Aug 2026 03:12:18 +0800 Subject: [PATCH 06/18] fix(proxy): keep durable replay winners visible on concurrent conflicts When persistCompleted detected an existing matching durable row or hit a conflict during concurrent writes, the spool previously wrote an aborted terminal state to the hot layer, masking the already-persisted PG winner. persistCompleted now returns a discriminated result so the spool can detect existing winners, and conflicts raise a dedicated ReplayDurableConflictError. In both cases the spool calls the new discardOwned method to silently drop the losing hot-layer candidate without writing an aborted status. The sourceMessageRequestId check was also relaxed from the persisted-row equality test so replays of the same content from different source requests are accepted. --- src/app/v1/_lib/proxy/replay/replay-spool.ts | 23 ++++++++- src/app/v1/_lib/proxy/replay/replay-store.ts | 49 ++++++++++++++++++-- tests/unit/proxy/replay-spool.test.ts | 37 ++++++++++++++- tests/unit/proxy/replay-store.test.ts | 33 +++++++++++-- 4 files changed, 130 insertions(+), 12 deletions(-) diff --git a/src/app/v1/_lib/proxy/replay/replay-spool.ts b/src/app/v1/_lib/proxy/replay/replay-spool.ts index 73303eb1c..c14ccdbae 100644 --- a/src/app/v1/_lib/proxy/replay/replay-spool.ts +++ b/src/app/v1/_lib/proxy/replay/replay-spool.ts @@ -3,7 +3,12 @@ import { logger } from "@/lib/logger"; import type { ProxySession } from "../session"; import { captureReplayResponseHeaders } from "./replay-headers"; import { isReplayEnabled, type ReplayIdentity } from "./replay-identity"; -import { getReplayStore, type ReplayDelivery, type ReplayMeta } from "./replay-store"; +import { + getReplayStore, + type ReplayDelivery, + ReplayDurableConflictError, + type ReplayMeta, +} from "./replay-store"; /** * F2 owner 侧 spool：把客户端可见字节（pump 处理后流）以 write-behind 方式 @@ -251,7 +256,7 @@ export class ReplaySpool { this.chunkCount = appended; this.metaWritten = true; // 先写 PG（持久 payload），再翻 Redis meta 为 completed（热层可服务） - await this.store.persistCompleted({ + const persistResult = await this.store.persistCompleted({ replayId: this.identity.replayId, verifier: this.identity.verifier, scopeTag: this.identity.scopeTag, @@ -265,6 +270,13 @@ export class ReplaySpool { byteSize: this.totalBytes, sourceMessageRequestId: messageRequestId, }); + if (persistResult === "existing") { + await this.store.discardOwned(this.identity.replayId, this.ownerToken); + logger.info("[ReplaySpool] reused existing durable replay winner", { + replayId: this.identity.replayId.slice(0, 12), + }); + return; + } pgPersisted = true; const completed = await this.store.completeOwned( this.identity.replayId, @@ -280,6 +292,13 @@ export class ReplaySpool { byteSize: this.totalBytes, }); } catch (error) { + if (error instanceof ReplayDurableConflictError) { + logger.warn("[ReplaySpool] discarded conflicting durable replay candidate", { + replayId: this.identity.replayId.slice(0, 12), + }); + await this.store.discardOwned(this.identity.replayId, this.ownerToken).catch(() => false); + return; + } // pgPersisted=true：payload 已 durable，仅 completed 翻转失败——热层封死为 // aborted 仍正确（meta 过期后可由 PG 持久层继续服务）；false 则未持久化，整体作废 logger.warn("[ReplaySpool] complete failed, aborting entry", { diff --git a/src/app/v1/_lib/proxy/replay/replay-store.ts b/src/app/v1/_lib/proxy/replay/replay-store.ts index 56b8f581b..a58875a96 100644 --- a/src/app/v1/_lib/proxy/replay/replay-store.ts +++ b/src/app/v1/_lib/proxy/replay/replay-store.ts @@ -81,6 +81,15 @@ redis.call('DEL', KEYS[3]) redis.call('DEL', KEYS[1]) return 1`; +const LUA_DISCARD_OWNED = ` +if redis.call('GET', KEYS[1]) ~= ARGV[1] then + return 0 +end +redis.call('DEL', KEYS[2]) +redis.call('DEL', KEYS[3]) +redis.call('DEL', KEYS[1]) +return 1`; + const LUA_COMPLETE_OWNED = ` if redis.call('GET', KEYS[1]) ~= ARGV[1] then return 0 @@ -108,6 +117,13 @@ export interface ReplayPersistedRow { sourceMessageRequestId: number | null; } +export class ReplayDurableConflictError extends Error { + constructor(replayId: string) { + super(`durable replay conflict for ${replayId.slice(0, 12)}`); + this.name = "ReplayDurableConflictError"; + } +} + export const REPLAY_CLEANUP_BATCH_SIZE = 100; function hasMatchingHeaders( @@ -137,8 +153,7 @@ function isMatchingPersistedReplay( actual.statusCode === expected.statusCode && hasMatchingHeaders(expected.headers, actual.headersJson) && actual.payload === expected.payload && - actual.byteSize === expected.byteSize && - actual.sourceMessageRequestId === expected.sourceMessageRequestId + actual.byteSize === expected.byteSize ); } @@ -303,6 +318,29 @@ export class ReplayStore { } } + /** 当前 owner 放弃热层候选，但不写 aborted，避免遮蔽已存在的 PG winner。 */ + async discardOwned(replayId: string, ownerToken: string): Promise { + const redis = this.getRawRedis(); + if (!redis) return false; + try { + const result = await redis.eval( + LUA_DISCARD_OWNED, + 3, + `cch:replay:owner:${replayId}`, + `cch:replay:meta:${replayId}`, + `cch:replay:chunks:${replayId}`, + ownerToken + ); + return result === 1; + } catch (error) { + logger.debug("[ReplayStore] fenced discard failed", { + replayId: replayId.slice(0, 12), + error: error instanceof Error ? error.message : String(error), + }); + return false; + } + } + /** 仅当前 token 仍持有租约时，原子翻转 completed meta 并释放租约。 */ async completeOwned(replayId: string, ownerToken: string, meta: ReplayMeta): Promise { const redis = this.getRawRedis(); @@ -334,7 +372,7 @@ export class ReplayStore { * 走 abort——payload 未 durable 时绝不能把 meta 翻成 completed。 * （过期行清理由 instrumentation 定时调度器负责，不在写路径顺带执行。） */ - async persistCompleted(row: ReplayPersistedRow): Promise { + async persistCompleted(row: ReplayPersistedRow): Promise<"persisted" | "existing"> { const env = getEnvConfig(); const now = new Date(); const expiresAt = new Date(now.getTime() + env.REPLAY_COMPLETED_TTL_SECONDS * 1000); @@ -369,7 +407,7 @@ export class ReplayStore { }) .returning({ replayId: replayPayloads.replayId }); - if (upserted.length > 0) return; + if (upserted.length > 0) return "persisted"; const existingRows = await db .select() @@ -378,8 +416,9 @@ export class ReplayStore { .limit(1); const existing = existingRows[0]; if (!existing || !isMatchingPersistedReplay(row, existing)) { - throw new Error(`durable replay conflict for ${row.replayId.slice(0, 12)}`); + throw new ReplayDurableConflictError(row.replayId); } + return "existing"; } catch (error) { logger.warn("[ReplayStore] persistCompleted failed", { error: error instanceof Error ? error.message : String(error), diff --git a/tests/unit/proxy/replay-spool.test.ts b/tests/unit/proxy/replay-spool.test.ts index 07dcdfe35..3e502420d 100644 --- a/tests/unit/proxy/replay-spool.test.ts +++ b/tests/unit/proxy/replay-spool.test.ts @@ -6,7 +6,10 @@ import { getActiveReplaySpoolCount, ReplaySpool, } from "@/app/v1/_lib/proxy/replay/replay-spool"; -import type { ReplayDelivery } from "@/app/v1/_lib/proxy/replay/replay-store"; +import { + ReplayDurableConflictError, + type ReplayDelivery, +} from "@/app/v1/_lib/proxy/replay/replay-store"; import type { ProxySession } from "@/app/v1/_lib/proxy/session"; import { logger } from "@/lib/logger"; @@ -64,6 +67,11 @@ const storeControl = vi.hoisted(() => { }), persistCompleted: vi.fn(async () => { order.push("persist"); + return "persisted" as const; + }), + discardOwned: vi.fn(async () => { + order.push("discard"); + return true; }), deleteEntry: vi.fn(async () => { order.push("deleteEntry"); @@ -114,6 +122,7 @@ vi.mock("@/lib/config/env.schema", async (importOriginal) => { vi.mock("@/app/v1/_lib/proxy/replay/replay-store", () => ({ getReplayStore: () => storeControl.store, + ReplayDurableConflictError: class ReplayDurableConflictError extends Error {}, })); const identity: ReplayIdentity = { @@ -431,6 +440,32 @@ describe("ReplaySpool：completeAfterBilling 终态屏障", () => { expect(getActiveReplaySpoolCount()).toBe(0); }); + it("复用已有 durable winner 时丢弃当前热层候选，不写 aborted", async () => { + storeControl.store.persistCompleted.mockResolvedValueOnce("existing"); + const spool = makeSpool(); + spool.observe(encoder.encode("data: a\n\n")); + + await spool.completeAfterBilling(7); + + expect(storeControl.store.discardOwned).toHaveBeenCalledWith(identity.replayId, "owner-token"); + expect(storeControl.store.completeOwned).not.toHaveBeenCalled(); + expect(storeControl.store.abortOwned).not.toHaveBeenCalled(); + }); + + it("durable payload 冲突时丢弃当前候选，不用 aborted 遮蔽 PG winner", async () => { + storeControl.store.persistCompleted.mockRejectedValueOnce( + new ReplayDurableConflictError(identity.replayId) + ); + const spool = makeSpool(); + spool.observe(encoder.encode("data: a\n\n")); + + await spool.completeAfterBilling(7); + + expect(storeControl.store.discardOwned).toHaveBeenCalledWith(identity.replayId, "owner-token"); + expect(storeControl.store.completeOwned).not.toHaveBeenCalled(); + expect(storeControl.store.abortOwned).not.toHaveBeenCalled(); + }); + it("尾批 fenced write 返回 null（Redis 不可用）时终止为 aborted，绝不置 completed 也不写 PG", async () => { storeControl.store.writeOwned.mockResolvedValueOnce(null); const spool = makeSpool(); diff --git a/tests/unit/proxy/replay-store.test.ts b/tests/unit/proxy/replay-store.test.ts index b698894b7..aa9d4c6ed 100644 --- a/tests/unit/proxy/replay-store.test.ts +++ b/tests/unit/proxy/replay-store.test.ts @@ -186,6 +186,14 @@ function createFakeRedis() { kv.delete(key); return Number(ttl) > 0 ? 1 : 0; } + if (script.includes("KEYS[3]") && script.includes("DEL") && _numkeys === 3) { + const [metaKey, chunksKey, token] = args; + if (kv.get(key) !== token) return 0; + kv.delete(String(metaKey)); + lists.delete(String(chunksKey)); + kv.delete(key); + return 1; + } const token = args[0] as string; if (script.includes("EXPIRE")) { return kv.get(key) === token ? 1 : 0; @@ -518,6 +526,22 @@ describe("ReplayStore：owner 租约", () => { expect(currentRedis().kv.get("cch:replay:owner:r1")).toBe("tok-new"); }); + it("discardOwned 仅在 token 匹配时删除当前热层候选且不写终态", async () => { + const store = new ReplayStore(); + await store.tryClaimOwner("r1", "tok-a"); + await store.setMeta("r1", makeMeta()); + await store.appendChunks("r1", ["partial"]); + + await expect(store.discardOwned("r1", "tok-other")).resolves.toBe(false); + await expect(store.getMeta("r1")).resolves.not.toBeNull(); + await expect(store.readChunks("r1", 0)).resolves.toEqual(["partial"]); + + await expect(store.discardOwned("r1", "tok-a")).resolves.toBe(true); + await expect(store.getMeta("r1")).resolves.toBeNull(); + await expect(store.readChunks("r1", 0)).resolves.toEqual([]); + expect(currentRedis().kv.has("cch:replay:owner:r1")).toBe(false); + }); + it("completeOwned 仅在 token 匹配时原子写 completed meta 并释放租约", async () => { const store = new ReplayStore(); const completedMeta = makeMeta({ status: "completed", chunkCount: 2 }); @@ -540,7 +564,7 @@ describe("ReplayStore：PG 完成持久层", () => { const row = makePersistedRow(); const before = Date.now(); - await store.persistCompleted(row); + await expect(store.persistCompleted(row)).resolves.toBe("persisted"); const after = Date.now(); expect(dbState.insertValues).toHaveLength(1); @@ -572,7 +596,7 @@ describe("ReplayStore：PG 完成持久层", () => { const store = new ReplayStore(); const row = makePersistedRow({ payload: "data: replacement\n\n" }); - await expect(store.persistCompleted(row)).resolves.toBeUndefined(); + await expect(store.persistCompleted(row)).resolves.toBe("persisted"); expect(dbState.upsertConfigs).toHaveLength(1); const config = dbState.upsertConfigs[0] as { @@ -593,19 +617,20 @@ describe("ReplayStore：PG 完成持久层", () => { expect(dbState.selectWheres).toHaveLength(0); }); - it("persistCompleted 接受内容完全一致的未过期 durable 行", async () => { + it("persistCompleted 接受内容一致但 source request 不同的未过期 durable 行", async () => { dbState.upsertRows = []; const row = makePersistedRow(); dbState.selectRows = [ { ...row, headersJson: row.headers, + sourceMessageRequestId: 999, expiresAt: new Date(Date.now() + 60_000), }, ]; const store = new ReplayStore(); - await expect(store.persistCompleted(row)).resolves.toBeUndefined(); + await expect(store.persistCompleted(row)).resolves.toBe("existing"); expect(dbState.selectWheres).toHaveLength(1); }); From 72be4680a7d674bf1e79c6c5763a341eda133942 Mon Sep 17 00:00:00 2001 From: ding113 Date: Mon, 3 Aug 2026 03:12:18 +0800 Subject: [PATCH 07/18] feat(reset): preserve fixed 5h cost windows and recover ambiguous enqueues Statistics resets previously wiped all Redis cost-cache keys including the fixed 5h rolling windows, causing post-cutoff quota state to be lost. A new prepareUserStatisticsResetFixed5h function atomically deletes only the fixed 5h keys and records a monotonic cutoff timestamp, and clearUserCostCache now accepts preserveFixed5hCostKeys to skip those keys during the broader cleanup. The reset queue now runs this preparation phase before enqueuing and tracks a preparation version on the stored record so legacy queued jobs are reconciled idempotently on retry. Enqueue failures after preparation leave the record queued rather than discarding it, and the next enqueue attempt recovers the prepared claim. Per-batch deletion progress is persisted to the status store so retries cumulate instead of restarting, and active-claim release errors no longer mask the original business error code. --- src/actions/users.ts | 4 +- src/lib/redis/cost-cache-cleanup.ts | 47 +++- src/lib/user-statistics-reset/reset-queue.ts | 257 +++++++++++++++--- .../user-statistics-reset/reset-service.ts | 42 ++- .../reset-status-store.ts | 15 +- src/lib/user-statistics-reset/types.ts | 7 + .../unit/actions/users-reset-5h-only.test.ts | 4 +- .../users-reset-all-statistics.test.ts | 4 +- .../unit/lib/redis/cost-cache-cleanup.test.ts | 53 ++++ .../lib/user-statistics-reset-queue.test.ts | 247 ++++++++++++++++- .../lib/user-statistics-reset-service.test.ts | 18 ++ ...user-statistics-reset-status-store.test.ts | 17 ++ 12 files changed, 659 insertions(+), 56 deletions(-) diff --git a/src/actions/users.ts b/src/actions/users.ts index d5f539f61..58c05a28e 100644 --- a/src/actions/users.ts +++ b/src/actions/users.ts @@ -2385,7 +2385,9 @@ export async function resetUserAllStatistics( const { enqueueUserStatisticsReset } = await import("@/lib/user-statistics-reset/reset-queue"); enqueueStarted = true; - const reset = await enqueueUserStatisticsReset(userId); + const reset = await enqueueUserStatisticsReset(userId, { + fixed5hKeyIds: keys.map((key) => key.id), + }); logger.info("Queued user statistics reset", { userId, resetId: reset.resetId, diff --git a/src/lib/redis/cost-cache-cleanup.ts b/src/lib/redis/cost-cache-cleanup.ts index e7a64bde9..6a13d5451 100644 --- a/src/lib/redis/cost-cache-cleanup.ts +++ b/src/lib/redis/cost-cache-cleanup.ts @@ -10,6 +10,7 @@ export interface ClearUserCostCacheOptions { keyHashes: string[]; includeActiveSessions?: boolean; allowWhenRateLimitDisabled?: boolean; + preserveFixed5hCostKeys?: boolean; } export interface ClearUserCostCacheResult { @@ -42,6 +43,47 @@ export interface ClearUser5hCostCacheResult { errorCount?: number; } +const STATISTICS_RESET_PREPARE_TTL_SECONDS = 7 * 24 * 60 * 60; +const PREPARE_FIXED_5H_RESET_LUA = ` +if redis.call('EXISTS', KEYS[1]) == 1 then + return redis.call('GET', KEYS[1]) +end +local now = redis.call('TIME') +local cutoff_ms = (tonumber(now[1]) * 1000) + math.floor(tonumber(now[2]) / 1000) +for index = 2, #KEYS do + redis.call('DEL', KEYS[index]) +end +redis.call('SETEX', KEYS[1], ARGV[1], tostring(cutoff_ms)) +return tostring(cutoff_ms)`; + +export async function prepareUserStatisticsResetFixed5h(input: { + resetId: string; + userId: number; + keyIds: number[]; +}): Promise { + const redis = getRedisClient({ allowWhenRateLimitDisabled: true }); + if (redis?.status !== "ready") return null; + + const keys = [ + `cch:user-statistics-reset:fixed5h:${input.resetId}`, + `user:${input.userId}:cost_5h_fixed`, + buildLeaseKey("user", input.userId, "5h", "fixed"), + ...input.keyIds.flatMap((keyId) => [ + `key:${keyId}:cost_5h_fixed`, + buildLeaseKey("key", keyId, "5h", "fixed"), + ]), + ]; + + const cutoffMilliseconds = await redis.eval( + PREPARE_FIXED_5H_RESET_LUA, + keys.length, + ...keys, + STATISTICS_RESET_PREPARE_TTL_SECONDS + ); + const cutoff = new Date(Number(cutoffMilliseconds)); + return Number.isFinite(cutoff.getTime()) ? cutoff.toISOString() : null; +} + /** * Scan and delete all Redis cost-cache keys for a user and their API keys. * @@ -59,6 +101,7 @@ export async function clearUserCostCache( keyHashes, includeActiveSessions = false, allowWhenRateLimitDisabled = false, + preserveFixed5hCostKeys = false, } = options; const redis = getRedisClient({ allowWhenRateLimitDisabled }); @@ -143,7 +186,9 @@ export async function clearUserCostCache( }), ]); - const allCostKeys = scanResults.flat(); + const allCostKeys = scanResults + .flat() + .filter((key) => !preserveFixed5hCostKeys || !key.endsWith(":cost_5h_fixed")); let activeSessionsDeleted = 0; // Only create pipeline if there is work to do diff --git a/src/lib/user-statistics-reset/reset-queue.ts b/src/lib/user-statistics-reset/reset-queue.ts index 619e4303b..3bb1f5ca9 100644 --- a/src/lib/user-statistics-reset/reset-queue.ts +++ b/src/lib/user-statistics-reset/reset-queue.ts @@ -5,7 +5,12 @@ import type { Job } from "bull"; import Queue from "bull"; import { logger } from "@/lib/logger"; import { buildRedisQueueOptions } from "@/lib/redis/bull-queue-options"; -import { executeUserStatisticsReset, UserStatisticsResetError } from "./reset-service"; +import { prepareUserStatisticsResetFixed5h } from "@/lib/redis/cost-cache-cleanup"; +import { + executeUserStatisticsReset, + findUserStatisticsResetKeyIds, + UserStatisticsResetError, +} from "./reset-service"; import { claimActiveUserStatisticsReset, deleteUserStatisticsResetStatus, @@ -13,7 +18,11 @@ import { releaseActiveUserStatisticsReset, setUserStatisticsResetStatus, } from "./reset-status-store"; -import type { UserStatisticsResetJobData, UserStatisticsResetRecord } from "./types"; +import type { + UserStatisticsResetJobData, + UserStatisticsResetRecord, + UserStatisticsResetStoredRecord, +} from "./types"; let resetQueue: Queue.Queue | null = null; const RESET_JOB_NAME = "reset"; @@ -25,9 +34,11 @@ function errorCode(error: unknown): string { : "USER_STATISTICS_RESET_OPERATION_FAILED"; } -function createQueuedRecord(input: UserStatisticsResetJobData): UserStatisticsResetRecord { +function createQueuedRecord(input: UserStatisticsResetJobData): UserStatisticsResetStoredRecord { return { ...input, + fixed5hKeyIds: input.fixed5hKeyIds ?? [], + fixed5hPreparationVersion: input.fixed5hPreparationVersion ?? null, status: "queued", startedAt: null, completedAt: null, @@ -37,6 +48,79 @@ function createQueuedRecord(input: UserStatisticsResetJobData): UserStatisticsRe }; } +function toPublicRecord(record: UserStatisticsResetStoredRecord): UserStatisticsResetRecord { + const { + fixed5hKeyIds: _fixed5hKeyIds, + fixed5hPreparationVersion: _fixed5hPreparationVersion, + ...publicRecord + } = record; + return publicRecord; +} + +type PreparedResetJobData = UserStatisticsResetJobData & { + fixed5hKeyIds: number[]; + fixed5hPreparationVersion: 1; +}; + +async function ensurePreparedReset( + jobData: UserStatisticsResetJobData, + current: UserStatisticsResetStoredRecord +): Promise<{ jobData: PreparedResetJobData; record: UserStatisticsResetStoredRecord }> { + if (current.fixed5hPreparationVersion === 1) { + return { + jobData: { + resetId: current.resetId, + userId: current.userId, + requestedAt: current.requestedAt, + fixed5hKeyIds: current.fixed5hKeyIds, + fixed5hPreparationVersion: 1, + }, + record: current, + }; + } + + if (jobData.fixed5hPreparationVersion === 1) { + const preparedRecord: UserStatisticsResetStoredRecord = { + ...current, + requestedAt: jobData.requestedAt, + fixed5hKeyIds: jobData.fixed5hKeyIds ?? [], + fixed5hPreparationVersion: 1, + }; + await setUserStatisticsResetStatus(preparedRecord); + return { + jobData: { + resetId: jobData.resetId, + userId: jobData.userId, + requestedAt: jobData.requestedAt, + fixed5hKeyIds: preparedRecord.fixed5hKeyIds, + fixed5hPreparationVersion: 1, + }, + record: preparedRecord, + }; + } + + const fixed5hKeyIds = await findUserStatisticsResetKeyIds(jobData.userId); + const requestedAt = await prepareUserStatisticsResetFixed5h({ + resetId: jobData.resetId, + userId: jobData.userId, + keyIds: fixed5hKeyIds, + }); + if (!requestedAt) throw new Error("USER_STATISTICS_RESET_FIXED_5H_PREPARE_FAILED"); + + const preparedJobData: PreparedResetJobData = { + ...jobData, + requestedAt, + fixed5hKeyIds, + fixed5hPreparationVersion: 1, + }; + const preparedRecord: UserStatisticsResetStoredRecord = { + ...current, + ...preparedJobData, + }; + await setUserStatisticsResetStatus(preparedRecord); + return { jobData: preparedJobData, record: preparedRecord }; +} + function getResetQueue(): Queue.Queue { if (resetQueue) return resetQueue; const redisUrl = process.env.REDIS_URL; @@ -88,75 +172,135 @@ async function recordFinalFailure( ...current, status: "failed", completedAt: new Date().toISOString(), - errorCode: errorCode(error), + errorCode: + current.status === "failed" && current.errorCode ? current.errorCode : errorCode(error), }); - await releaseActiveUserStatisticsReset(jobData.userId, jobData.resetId); + try { + await releaseActiveUserStatisticsReset(jobData.userId, jobData.resetId); + } catch (releaseError) { + logger.warn("[UserStatisticsResetQueue] failed reset retained an active claim", { + resetId: jobData.resetId, + userId: jobData.userId, + error: releaseError instanceof Error ? releaseError.message : String(releaseError), + }); + } } async function processUserStatisticsReset(job: Job) { let current = (await getUserStatisticsResetStatus(job.data.resetId)) ?? createQueuedRecord(job.data); const startedAt = current.startedAt ?? new Date().toISOString(); - current = { - ...current, - status: "running", - startedAt, - errorCode: null, + let baseProgress = { + deletedMessageRequests: current.deletedMessageRequests, + deletedUsageLedger: current.deletedUsageLedger, }; + let attemptProgress = { deletedMessageRequests: 0, deletedUsageLedger: 0 }; + let preparedJobData: PreparedResetJobData | null = null; try { + const prepared = await ensurePreparedReset(job.data, current); + preparedJobData = prepared.jobData; + current = { + ...prepared.record, + status: "running", + startedAt, + errorCode: null, + }; + baseProgress = { + deletedMessageRequests: current.deletedMessageRequests, + deletedUsageLedger: current.deletedUsageLedger, + }; await setUserStatisticsResetStatus(current); - const deleted = await executeUserStatisticsReset(job.data); - const completed: UserStatisticsResetRecord = { + const deleted = await executeUserStatisticsReset(preparedJobData, async (progress) => { + attemptProgress = progress; + current = { + ...current, + deletedMessageRequests: + baseProgress.deletedMessageRequests + progress.deletedMessageRequests, + deletedUsageLedger: baseProgress.deletedUsageLedger + progress.deletedUsageLedger, + }; + await setUserStatisticsResetStatus(current); + }); + attemptProgress = deleted; + const completed: UserStatisticsResetStoredRecord = { ...current, - deletedMessageRequests: current.deletedMessageRequests + deleted.deletedMessageRequests, - deletedUsageLedger: current.deletedUsageLedger + deleted.deletedUsageLedger, + deletedMessageRequests: baseProgress.deletedMessageRequests + deleted.deletedMessageRequests, + deletedUsageLedger: baseProgress.deletedUsageLedger + deleted.deletedUsageLedger, status: "completed", startedAt, completedAt: new Date().toISOString(), errorCode: null, }; await setUserStatisticsResetStatus(completed); - await releaseActiveUserStatisticsReset(job.data.userId, job.data.resetId); + try { + await releaseActiveUserStatisticsReset(job.data.userId, job.data.resetId); + } catch (error) { + logger.warn("[UserStatisticsResetQueue] completed reset retained an active claim", { + resetId: job.data.resetId, + userId: job.data.userId, + error: error instanceof Error ? error.message : String(error), + }); + } return completed; } catch (error) { const attempts = job.opts.attempts ?? 1; const isFinalAttempt = job.attemptsMade + 1 >= attempts; - const progress = + const errorProgress = error instanceof UserStatisticsResetError ? error.progress : { deletedMessageRequests: 0, deletedUsageLedger: 0 }; + const progress = { + deletedMessageRequests: Math.max( + attemptProgress.deletedMessageRequests, + errorProgress.deletedMessageRequests + ), + deletedUsageLedger: Math.max( + attemptProgress.deletedUsageLedger, + errorProgress.deletedUsageLedger + ), + }; await setUserStatisticsResetStatus({ ...current, - deletedMessageRequests: current.deletedMessageRequests + progress.deletedMessageRequests, - deletedUsageLedger: current.deletedUsageLedger + progress.deletedUsageLedger, + deletedMessageRequests: baseProgress.deletedMessageRequests + progress.deletedMessageRequests, + deletedUsageLedger: baseProgress.deletedUsageLedger + progress.deletedUsageLedger, status: isFinalAttempt ? "failed" : "queued", startedAt, completedAt: isFinalAttempt ? new Date().toISOString() : null, errorCode: isFinalAttempt ? errorCode(error) : null, }); if (isFinalAttempt) { - await releaseActiveUserStatisticsReset(job.data.userId, job.data.resetId); + try { + await releaseActiveUserStatisticsReset(job.data.userId, job.data.resetId); + } catch (releaseError) { + logger.warn("[UserStatisticsResetQueue] failed reset retained an active claim", { + resetId: job.data.resetId, + userId: job.data.userId, + error: releaseError instanceof Error ? releaseError.message : String(releaseError), + }); + } } throw error; } } export async function enqueueUserStatisticsReset( - userId: number + userId: number, + options: { requestedAt?: string; fixed5hKeyIds?: number[] } = {} ): Promise { - return enqueueUserStatisticsResetWithReconciliation(userId, true); + return enqueueUserStatisticsResetWithReconciliation(userId, options, true); } async function enqueueUserStatisticsResetWithReconciliation( userId: number, + options: { requestedAt?: string; fixed5hKeyIds?: number[] }, allowReconciliation: boolean ): Promise { const queue = getResetQueue(); const jobData: UserStatisticsResetJobData = { resetId: randomUUID(), userId, - requestedAt: new Date().toISOString(), + requestedAt: options.requestedAt ?? new Date().toISOString(), + fixed5hKeyIds: options.fixed5hKeyIds ?? [], }; const queued = createQueuedRecord(jobData); await setUserStatisticsResetStatus(queued); @@ -170,7 +314,7 @@ async function enqueueUserStatisticsResetWithReconciliation( if (!existingIsActive) { await releaseActiveUserStatisticsReset(userId, claim.resetId); if (allowReconciliation) { - return enqueueUserStatisticsResetWithReconciliation(userId, false); + return enqueueUserStatisticsResetWithReconciliation(userId, options, false); } throw new Error("USER_STATISTICS_RESET_ACTIVE_STATUS_MISSING"); } @@ -180,35 +324,74 @@ async function enqueueUserStatisticsResetWithReconciliation( if (existingJobState === "failed" || existingJobState === "completed") { await releaseActiveUserStatisticsReset(userId, existing.resetId); if (allowReconciliation) { - return enqueueUserStatisticsResetWithReconciliation(userId, false); + return enqueueUserStatisticsResetWithReconciliation(userId, options, false); } throw new Error("USER_STATISTICS_RESET_ACTIVE_JOB_TERMINAL"); } if (!existingJob) { - await queue.add( - RESET_JOB_NAME, + const recovered = await ensurePreparedReset( { resetId: existing.resetId, userId: existing.userId, requestedAt: existing.requestedAt, + fixed5hKeyIds: existing.fixed5hKeyIds, + ...(existing.fixed5hPreparationVersion === 1 + ? { fixed5hPreparationVersion: 1 as const } + : {}), }, - { jobId: existing.resetId } + existing ); + await queue.add(RESET_JOB_NAME, recovered.jobData, { jobId: existing.resetId }); + return toPublicRecord(recovered.record); } - return existing; + return toPublicRecord(existing); } + let preparedRecord: UserStatisticsResetStoredRecord | null = null; + let enqueueAttempted = false; try { - await queue.add(RESET_JOB_NAME, jobData, { jobId: jobData.resetId }); - return queued; - } catch (error) { - await setUserStatisticsResetStatus({ + const requestedAt = await prepareUserStatisticsResetFixed5h({ + resetId: jobData.resetId, + userId, + keyIds: jobData.fixed5hKeyIds ?? [], + }); + if (!requestedAt) throw new Error("USER_STATISTICS_RESET_FIXED_5H_PREPARE_FAILED"); + const preparedJobData: PreparedResetJobData = { + ...jobData, + requestedAt, + fixed5hKeyIds: jobData.fixed5hKeyIds ?? [], + fixed5hPreparationVersion: 1, + }; + preparedRecord = { ...queued, - status: "failed", - completedAt: new Date().toISOString(), - errorCode: "USER_STATISTICS_RESET_QUEUE_FAILED", + ...preparedJobData, + }; + await setUserStatisticsResetStatus(preparedRecord); + enqueueAttempted = true; + await queue.add(RESET_JOB_NAME, preparedJobData, { jobId: jobData.resetId }); + return toPublicRecord(preparedRecord); + } catch (error) { + if (enqueueAttempted && preparedRecord) { + try { + if (await queue.getJob(jobData.resetId)) { + return toPublicRecord(preparedRecord); + } + } catch (reconciliationError) { + logger.warn("[UserStatisticsResetQueue] ambiguous enqueue could not be reconciled", { + resetId: jobData.resetId, + userId, + error: + reconciliationError instanceof Error + ? reconciliationError.message + : String(reconciliationError), + }); + } + } + logger.warn("[UserStatisticsResetQueue] reset remains queued after enqueue failure", { + resetId: jobData.resetId, + userId, + error: error instanceof Error ? error.message : String(error), }); - await releaseActiveUserStatisticsReset(userId, jobData.resetId); throw error; } } @@ -218,7 +401,7 @@ export async function findUserStatisticsReset( resetId: string ): Promise { const record = await getUserStatisticsResetStatus(resetId); - return record?.userId === userId ? record : null; + return record?.userId === userId ? toPublicRecord(record) : null; } export function startUserStatisticsResetQueue(): boolean { diff --git a/src/lib/user-statistics-reset/reset-service.ts b/src/lib/user-statistics-reset/reset-service.ts index 514f1c05e..4384af7b6 100644 --- a/src/lib/user-statistics-reset/reset-service.ts +++ b/src/lib/user-statistics-reset/reset-service.ts @@ -8,6 +8,14 @@ import { invalidateCachedUser } from "@/lib/security/api-key-auth-cache"; const RESET_BATCH_SIZE = 1000; +export async function findUserStatisticsResetKeyIds(userId: number): Promise { + const userKeys = await db + .select({ id: keys.id }) + .from(keys) + .where(and(eq(keys.userId, userId), isNull(keys.deletedAt))); + return userKeys.map((key) => key.id); +} + export class UserStatisticsResetError extends Error { constructor( readonly code: string, @@ -103,6 +111,7 @@ async function drainTable(input: { table: "message_request" | "usage_ledger"; userId: number; cutoff: Date; + onProgress?: (deleted: number) => Promise; }): Promise { let deleted = 0; try { @@ -112,6 +121,9 @@ async function drainTable(input: { ? await deleteMessageRequestBatch(input.userId, input.cutoff) : await deleteUsageLedgerBatch(input.userId, input.cutoff); deleted += batchDeleted; + if (batchDeleted > 0) { + await input.onProgress?.(deleted); + } if (batchDeleted < RESET_BATCH_SIZE) break; } @@ -135,10 +147,16 @@ async function drainTable(input: { } } -export async function executeUserStatisticsReset(input: { - userId: number; - requestedAt: string; -}): Promise<{ deletedMessageRequests: number; deletedUsageLedger: number }> { +export async function executeUserStatisticsReset( + input: { + userId: number; + requestedAt: string; + }, + onProgress?: (progress: { + deletedMessageRequests: number; + deletedUsageLedger: number; + }) => Promise +): Promise<{ deletedMessageRequests: number; deletedUsageLedger: number }> { const cutoff = new Date(input.requestedAt); if (!Number.isFinite(cutoff.getTime())) { throw new UserStatisticsResetError("USER_STATISTICS_RESET_INVALID_CUTOFF"); @@ -151,17 +169,28 @@ export async function executeUserStatisticsReset(input: { table: "message_request", userId: input.userId, cutoff, + onProgress: async (deleted) => { + deletedMessageRequests = deleted; + await onProgress?.({ deletedMessageRequests, deletedUsageLedger }); + }, }); deletedUsageLedger = await drainTable({ table: "usage_ledger", userId: input.userId, cutoff, + onProgress: async (deleted) => { + deletedUsageLedger = deleted; + await onProgress?.({ deletedMessageRequests, deletedUsageLedger }); + }, }); } catch (error) { if (error instanceof UserStatisticsResetError) { throw new UserStatisticsResetError(error.code, { - deletedMessageRequests: deletedMessageRequests + error.progress.deletedMessageRequests, - deletedUsageLedger: deletedUsageLedger + error.progress.deletedUsageLedger, + deletedMessageRequests: Math.max( + deletedMessageRequests, + error.progress.deletedMessageRequests + ), + deletedUsageLedger: Math.max(deletedUsageLedger, error.progress.deletedUsageLedger), }); } if (deletedMessageRequests > 0 || deletedUsageLedger > 0) { @@ -195,6 +224,7 @@ export async function executeUserStatisticsReset(input: { keyHashes: userKeys.map((key) => key.key), includeActiveSessions: false, allowWhenRateLimitDisabled: true, + preserveFixed5hCostKeys: true, }); if (!cacheResult || cacheResult.cleanupFailed) { throw new UserStatisticsResetError("USER_STATISTICS_RESET_CACHE_CLEANUP_FAILED"); diff --git a/src/lib/user-statistics-reset/reset-status-store.ts b/src/lib/user-statistics-reset/reset-status-store.ts index eb0ac3818..4dc10ebb8 100644 --- a/src/lib/user-statistics-reset/reset-status-store.ts +++ b/src/lib/user-statistics-reset/reset-status-store.ts @@ -3,12 +3,12 @@ import "server-only"; import type Redis from "ioredis"; import { getRedisClient } from "@/lib/redis/client"; import { RedisKVStore } from "@/lib/redis/redis-kv-store"; -import type { UserStatisticsResetRecord } from "./types"; +import type { UserStatisticsResetStoredRecord } from "./types"; const RESET_STATUS_TTL_SECONDS = 7 * 24 * 60 * 60; const ACTIVE_RESET_PREFIX = "cch:user-statistics-reset:active:"; const RESET_STATUS_PREFIX = "cch:user-statistics-reset:status:"; -const statusStore = new RedisKVStore({ +const statusStore = new RedisKVStore({ prefix: RESET_STATUS_PREFIX, defaultTtlSeconds: RESET_STATUS_TTL_SECONDS, }); @@ -32,7 +32,7 @@ function getReadyRedis(): ResetRedis { } export async function setUserStatisticsResetStatus( - record: UserStatisticsResetRecord + record: UserStatisticsResetStoredRecord ): Promise { if (!(await statusStore.set(record.resetId, record))) { throw new Error("USER_STATISTICS_RESET_STATUS_WRITE_FAILED"); @@ -41,11 +41,16 @@ export async function setUserStatisticsResetStatus( export async function getUserStatisticsResetStatus( resetId: string -): Promise { +): Promise { const raw = await getReadyRedis().get(`${RESET_STATUS_PREFIX}${resetId}`); if (!raw) return null; try { - return JSON.parse(raw) as UserStatisticsResetRecord; + const record = JSON.parse(raw) as UserStatisticsResetStoredRecord; + return { + ...record, + fixed5hKeyIds: record.fixed5hKeyIds ?? [], + fixed5hPreparationVersion: record.fixed5hPreparationVersion === 1 ? 1 : null, + }; } catch { throw new Error("USER_STATISTICS_RESET_STATUS_INVALID"); } diff --git a/src/lib/user-statistics-reset/types.ts b/src/lib/user-statistics-reset/types.ts index a0ce19e7e..dcceafcc5 100644 --- a/src/lib/user-statistics-reset/types.ts +++ b/src/lib/user-statistics-reset/types.ts @@ -16,4 +16,11 @@ export interface UserStatisticsResetJobData { resetId: string; userId: number; requestedAt: string; + fixed5hKeyIds?: number[]; + fixed5hPreparationVersion?: 1; +} + +export interface UserStatisticsResetStoredRecord extends UserStatisticsResetRecord { + fixed5hKeyIds: number[]; + fixed5hPreparationVersion: 1 | null; } diff --git a/tests/unit/actions/users-reset-5h-only.test.ts b/tests/unit/actions/users-reset-5h-only.test.ts index 6ab46a5e5..b451f8e76 100644 --- a/tests/unit/actions/users-reset-5h-only.test.ts +++ b/tests/unit/actions/users-reset-5h-only.test.ts @@ -322,7 +322,9 @@ describe("full reset compatibility with user 5h marker", () => { const result = await resetUserAllStatistics(123); expect(result.ok).toBe(true); - expect(enqueueUserStatisticsResetMock).toHaveBeenCalledWith(123); + expect(enqueueUserStatisticsResetMock).toHaveBeenCalledWith(123, { + fixed5hKeyIds: [11], + }); expect(txUpdateSetMock).not.toHaveBeenCalled(); expect(invalidateCachedUserMock).not.toHaveBeenCalled(); }); diff --git a/tests/unit/actions/users-reset-all-statistics.test.ts b/tests/unit/actions/users-reset-all-statistics.test.ts index 50a6930ae..3c38a6d61 100644 --- a/tests/unit/actions/users-reset-all-statistics.test.ts +++ b/tests/unit/actions/users-reset-all-statistics.test.ts @@ -95,7 +95,9 @@ describe("resetUserAllStatistics", () => { const result = await resetUserAllStatistics(123); expect(result).toEqual({ ok: true, data: queuedReset }); - expect(mocks.enqueue).toHaveBeenCalledWith(123); + expect(mocks.enqueue).toHaveBeenCalledWith(123, { + fixed5hKeyIds: [], + }); }); test("maps queue failures to a retryable dependency error", async () => { diff --git a/tests/unit/lib/redis/cost-cache-cleanup.test.ts b/tests/unit/lib/redis/cost-cache-cleanup.test.ts index b9e22a524..c046ea1a8 100644 --- a/tests/unit/lib/redis/cost-cache-cleanup.test.ts +++ b/tests/unit/lib/redis/cost-cache-cleanup.test.ts @@ -18,6 +18,7 @@ const redisPipelineMock = { const redisMock = { status: "ready" as string, pipeline: vi.fn(() => redisPipelineMock), + eval: vi.fn(), }; const getRedisClientMock = vi.fn(() => redisMock); vi.mock("@/lib/redis", () => ({ @@ -43,6 +44,7 @@ describe("clearUserCostCache", () => { getRedisClientMock.mockReturnValue(redisMock); redisMock.status = "ready"; redisMock.pipeline.mockReturnValue(redisPipelineMock); + redisMock.eval.mockResolvedValue(1); redisPipelineMock.del.mockReturnThis(); redisPipelineMock.exec.mockResolvedValue([]); scanPatternMock.mockResolvedValue([]); @@ -105,6 +107,57 @@ describe("clearUserCostCache", () => { expect(redisPipelineMock.exec).toHaveBeenCalled(); }); + test("preserves fixed 5h cost windows while invalidating other cost and lease keys", async () => { + scanPatternMock.mockImplementation(async (_redis: unknown, pattern: string) => { + if (pattern === "key:1:cost_*") { + return ["key:1:cost_5h_fixed", "key:1:cost_daily_rolling"]; + } + if (pattern === "user:10:cost_*") return ["user:10:cost_5h_fixed"]; + if (pattern === "lease:key:1:*") return ["lease:key:1:5h:fixed"]; + return []; + }); + redisPipelineMock.exec.mockResolvedValue([ + [null, 1], + [null, 1], + ]); + + const { clearUserCostCache } = await import("@/lib/redis/cost-cache-cleanup"); + const result = await clearUserCostCache({ + userId: 10, + keyIds: [1], + keyHashes: [], + preserveFixed5hCostKeys: true, + }); + + expect(result?.costKeysDeleted).toBe(2); + expect(redisPipelineMock.del).not.toHaveBeenCalledWith("key:1:cost_5h_fixed"); + expect(redisPipelineMock.del).not.toHaveBeenCalledWith("user:10:cost_5h_fixed"); + expect(redisPipelineMock.del).toHaveBeenCalledWith("key:1:cost_daily_rolling"); + expect(redisPipelineMock.del).toHaveBeenCalledWith("lease:key:1:5h:fixed"); + }); + + test("prepares fixed 5h reset atomically and idempotently by reset id", async () => { + redisMock.eval.mockResolvedValue(1_775_304_000_123); + const { prepareUserStatisticsResetFixed5h } = await import("@/lib/redis/cost-cache-cleanup"); + + await expect( + prepareUserStatisticsResetFixed5h({ resetId: "reset-1", userId: 10, keyIds: [1, 2] }) + ).resolves.toBe("2026-04-04T12:00:00.123Z"); + + expect(redisMock.eval).toHaveBeenCalledWith( + expect.stringMatching(/EXISTS[\s\S]*TIME[\s\S]*SETEX/), + 7, + "cch:user-statistics-reset:fixed5h:reset-1", + "user:10:cost_5h_fixed", + "lease:user:10:5h:fixed", + "key:1:cost_5h_fixed", + "lease:key:1:5h:fixed", + "key:2:cost_5h_fixed", + "lease:key:2:5h:fixed", + 604_800 + ); + }); + test("returns metrics (costKeysDeleted, activeSessionsDeleted, durationMs)", async () => { scanPatternMock.mockImplementation(async (_redis: unknown, pattern: string) => { if (pattern === "key:1:cost_*") return ["key:1:cost_daily"]; diff --git a/tests/unit/lib/user-statistics-reset-queue.test.ts b/tests/unit/lib/user-statistics-reset-queue.test.ts index 985ccef4c..350365a71 100644 --- a/tests/unit/lib/user-statistics-reset-queue.test.ts +++ b/tests/unit/lib/user-statistics-reset-queue.test.ts @@ -13,6 +13,8 @@ const boundary = vi.hoisted(() => ({ setStatus: vi.fn(), deleteStatus: vi.fn(), release: vi.fn(), + prepareFixed5h: vi.fn(), + findKeyIds: vi.fn(), execute: vi.fn(), })); @@ -36,6 +38,9 @@ vi.mock("bull", () => ({ vi.mock("@/lib/redis/bull-queue-options", () => ({ buildRedisQueueOptions: () => ({ host: "redis" }), })); +vi.mock("@/lib/redis/cost-cache-cleanup", () => ({ + prepareUserStatisticsResetFixed5h: boundary.prepareFixed5h, +})); vi.mock("@/lib/user-statistics-reset/reset-status-store", () => ({ claimActiveUserStatisticsReset: boundary.claim, getUserStatisticsResetStatus: boundary.getStatus, @@ -45,8 +50,12 @@ vi.mock("@/lib/user-statistics-reset/reset-status-store", () => ({ })); vi.mock("@/lib/user-statistics-reset/reset-service", () => ({ executeUserStatisticsReset: boundary.execute, + findUserStatisticsResetKeyIds: boundary.findKeyIds, UserStatisticsResetError: class UserStatisticsResetError extends Error { - constructor(readonly code: string) { + constructor( + readonly code: string, + readonly progress = { deletedMessageRequests: 0, deletedUsageLedger: 0 } + ) { super(code); } }, @@ -57,6 +66,7 @@ import { startUserStatisticsResetQueue, stopUserStatisticsResetQueue, } from "@/lib/user-statistics-reset/reset-queue"; +import { UserStatisticsResetError } from "@/lib/user-statistics-reset/reset-service"; const existing = { resetId: "00000000-0000-4000-8000-000000000002", @@ -68,6 +78,8 @@ const existing = { deletedMessageRequests: 1000, deletedUsageLedger: 0, errorCode: null, + fixed5hKeyIds: [9], + fixed5hPreparationVersion: 1 as const, }; describe("user statistics reset queue", () => { @@ -87,6 +99,8 @@ describe("user statistics reset queue", () => { boundary.setStatus, boundary.deleteStatus, boundary.release, + boundary.prepareFixed5h, + boundary.findKeyIds, boundary.execute, ]) mock.mockReset(); @@ -98,13 +112,17 @@ describe("user statistics reset queue", () => { boundary.setStatus.mockResolvedValue(undefined); boundary.deleteStatus.mockResolvedValue(undefined); boundary.release.mockResolvedValue(undefined); + boundary.prepareFixed5h.mockResolvedValue("2026-08-02T12:00:00.000Z"); + boundary.findKeyIds.mockResolvedValue([9]); }); it("returns the existing active reset instead of enqueueing a competitor", async () => { boundary.claim.mockResolvedValue({ acquired: false, resetId: existing.resetId }); boundary.getStatus.mockResolvedValue(existing); - await expect(enqueueUserStatisticsReset(42)).resolves.toEqual(existing); + await expect(enqueueUserStatisticsReset(42)).resolves.toEqual( + expect.not.objectContaining({ fixed5hKeyIds: expect.anything() }) + ); expect(boundary.deleteStatus).toHaveBeenCalledTimes(1); expect(boundary.add).not.toHaveBeenCalled(); }); @@ -136,19 +154,119 @@ describe("user statistics reset queue", () => { boundary.getStatus.mockResolvedValue(queued); boundary.getJob.mockResolvedValue(null); - await expect(enqueueUserStatisticsReset(42)).resolves.toEqual(queued); + await expect(enqueueUserStatisticsReset(42)).resolves.toEqual( + expect.not.objectContaining({ fixed5hKeyIds: expect.anything() }) + ); expect(boundary.add).toHaveBeenCalledWith( "reset", { resetId: queued.resetId, userId: queued.userId, - requestedAt: queued.requestedAt, + requestedAt: "2026-08-02T12:00:00.000Z", + fixed5hKeyIds: queued.fixed5hKeyIds, + fixed5hPreparationVersion: 1, }, { jobId: queued.resetId } ); }); + it("reconciles legacy queued records with current child key ids", async () => { + const { + fixed5hKeyIds: _fixed5hKeyIds, + fixed5hPreparationVersion: _fixed5hPreparationVersion, + ...legacyQueued + } = { + ...existing, + status: "queued" as const, + startedAt: null, + }; + boundary.claim.mockResolvedValue({ acquired: false, resetId: legacyQueued.resetId }); + boundary.getStatus.mockResolvedValue(legacyQueued); + boundary.getJob.mockResolvedValue(null); + + await expect(enqueueUserStatisticsReset(42)).resolves.toMatchObject({ + resetId: legacyQueued.resetId, + requestedAt: "2026-08-02T12:00:00.000Z", + }); + + expect(boundary.prepareFixed5h).toHaveBeenCalledWith({ + resetId: legacyQueued.resetId, + userId: 42, + keyIds: [9], + }); + expect(boundary.add).toHaveBeenCalledWith( + "reset", + expect.objectContaining({ fixed5hKeyIds: [9], fixed5hPreparationVersion: 1 }), + { jobId: legacyQueued.resetId } + ); + }); + + it("keeps a prepared claim recoverable when Bull enqueue fails", async () => { + let activeResetId = ""; + const storedRecords = new Map(); + boundary.claim.mockImplementation(async (_userId: number, resetId: string) => { + if (!activeResetId) { + activeResetId = resetId; + return { acquired: true, resetId }; + } + return { acquired: false, resetId: activeResetId }; + }); + boundary.setStatus.mockImplementation(async (record) => { + storedRecords.set(record.resetId, record); + }); + boundary.getStatus.mockImplementation(async (resetId: string) => storedRecords.get(resetId)); + boundary.getJob.mockResolvedValue(null); + boundary.add.mockRejectedValueOnce(new Error("redis timeout")).mockResolvedValueOnce({ + id: "recovered-job", + }); + + await expect(enqueueUserStatisticsReset(42, { fixed5hKeyIds: [9] })).rejects.toThrow( + "redis timeout" + ); + + expect(storedRecords.get(activeResetId)).toMatchObject({ + resetId: activeResetId, + status: "queued", + requestedAt: "2026-08-02T12:00:00.000Z", + fixed5hKeyIds: [9], + fixed5hPreparationVersion: 1, + }); + expect(boundary.release).not.toHaveBeenCalled(); + + await expect(enqueueUserStatisticsReset(42)).resolves.toMatchObject({ + resetId: activeResetId, + requestedAt: "2026-08-02T12:00:00.000Z", + }); + + expect(boundary.prepareFixed5h).toHaveBeenCalledTimes(1); + expect(boundary.add).toHaveBeenLastCalledWith( + "reset", + expect.objectContaining({ + resetId: activeResetId, + fixed5hKeyIds: [9], + fixed5hPreparationVersion: 1, + }), + { jobId: activeResetId } + ); + }); + + it("accepts an ambiguous enqueue when the deterministic Bull job exists", async () => { + boundary.claim.mockImplementation(async (_userId: number, resetId: string) => ({ + acquired: true, + resetId, + })); + boundary.add.mockRejectedValueOnce(new Error("connection closed")); + boundary.getJob.mockResolvedValueOnce({ id: "persisted-job" }); + + await expect(enqueueUserStatisticsReset(42, { fixed5hKeyIds: [9] })).resolves.toMatchObject({ + status: "queued", + requestedAt: "2026-08-02T12:00:00.000Z", + }); + + expect(boundary.release).not.toHaveBeenCalled(); + }); + it("releases a stale terminal claim and creates a new reset", async () => { boundary.claim .mockResolvedValueOnce({ acquired: false, resetId: existing.resetId }) @@ -168,6 +286,11 @@ describe("user statistics reset queue", () => { expect.objectContaining({ resetId: queued.resetId }), { jobId: queued.resetId } ); + expect(boundary.prepareFixed5h).toHaveBeenCalledWith({ + resetId: queued.resetId, + userId: 42, + keyIds: [], + }); }); it("releases a claim whose retained Bull job is already terminal", async () => { @@ -225,6 +348,72 @@ describe("user statistics reset queue", () => { expect(boundary.release).toHaveBeenCalledWith(42, queued.resetId); }); + it("prepares a legacy Bull job before deleting statistics", async () => { + startUserStatisticsResetQueue(); + const { + fixed5hKeyIds: _fixed5hKeyIds, + fixed5hPreparationVersion: _fixed5hPreparationVersion, + ...legacyJobData + } = existing; + const legacyStatus = { ...legacyJobData, status: "queued" as const, startedAt: null }; + boundary.getStatus.mockResolvedValue(legacyStatus); + boundary.findKeyIds.mockResolvedValue([9, 10]); + boundary.execute.mockResolvedValue({ + deletedMessageRequests: 2, + deletedUsageLedger: 3, + }); + + await boundary.processHandler?.({ + data: legacyJobData, + attemptsMade: 0, + opts: { attempts: 5 }, + }); + + expect(boundary.findKeyIds).toHaveBeenCalledWith(42); + expect(boundary.prepareFixed5h).toHaveBeenCalledWith({ + resetId: existing.resetId, + userId: 42, + keyIds: [9, 10], + }); + expect(boundary.execute).toHaveBeenCalledWith( + expect.objectContaining({ + requestedAt: "2026-08-02T12:00:00.000Z", + fixed5hKeyIds: [9, 10], + fixed5hPreparationVersion: 1, + }), + expect.any(Function) + ); + }); + + it("persists per-batch progress and does not regress completed state when claim release fails", async () => { + boundary.claim.mockImplementation(async (_userId: number, resetId: string) => ({ + acquired: true, + resetId, + })); + const queued = await enqueueUserStatisticsReset(42); + boundary.getStatus.mockResolvedValue({ ...queued, fixed5hKeyIds: [] }); + boundary.execute.mockImplementation(async (_data, onProgress) => { + await onProgress({ deletedMessageRequests: 1000, deletedUsageLedger: 0 }); + return { deletedMessageRequests: 1000, deletedUsageLedger: 5 }; + }); + boundary.release.mockRejectedValueOnce(new Error("redis unavailable")); + + await expect( + boundary.processHandler?.({ data: queued, attemptsMade: 0, opts: { attempts: 5 } }) + ).resolves.toEqual(expect.objectContaining({ status: "completed" })); + + expect(boundary.setStatus).toHaveBeenCalledWith( + expect.objectContaining({ status: "running", deletedMessageRequests: 1000 }) + ); + expect(boundary.setStatus).toHaveBeenLastCalledWith( + expect.objectContaining({ + status: "completed", + deletedMessageRequests: 1000, + deletedUsageLedger: 5, + }) + ); + }); + it("keeps retryable failures active and records a stable final error", async () => { boundary.claim.mockImplementation(async (_userId: number, resetId: string) => ({ acquired: true, @@ -255,6 +444,36 @@ describe("user statistics reset queue", () => { expect(boundary.release).toHaveBeenCalledWith(42, queued.resetId); }); + it("preserves the final business error when active claim release fails", async () => { + boundary.claim.mockImplementation(async (_userId: number, resetId: string) => ({ + acquired: true, + resetId, + })); + const queued = await enqueueUserStatisticsReset(42); + boundary.getStatus.mockResolvedValue({ + ...queued, + fixed5hKeyIds: [], + fixed5hPreparationVersion: 1, + }); + const resetError = new UserStatisticsResetError("USER_STATISTICS_RESET_CACHE_CLEANUP_FAILED", { + deletedMessageRequests: 5, + deletedUsageLedger: 7, + }); + boundary.execute.mockRejectedValue(resetError); + boundary.release.mockRejectedValueOnce(new Error("redis unavailable")); + + await expect( + boundary.processHandler?.({ data: queued, attemptsMade: 4, opts: { attempts: 5 } }) + ).rejects.toBe(resetError); + + expect(boundary.setStatus).toHaveBeenLastCalledWith( + expect.objectContaining({ + status: "failed", + errorCode: "USER_STATISTICS_RESET_CACHE_CLEANUP_FAILED", + }) + ); + }); + it("marks max-stalled jobs failed even when attemptsMade did not advance", async () => { startUserStatisticsResetQueue(); boundary.getStatus.mockResolvedValue(existing); @@ -276,4 +495,24 @@ describe("user statistics reset queue", () => { ); expect(boundary.release).toHaveBeenCalledWith(42, existing.resetId); }); + + it("keeps failed terminal status when releasing the active claim fails", async () => { + startUserStatisticsResetQueue(); + boundary.getStatus.mockResolvedValue(existing); + boundary.release.mockRejectedValueOnce(new Error("redis unavailable")); + + await expect( + boundary.failedHandler?.( + { data: existing, attemptsMade: 5, opts: { attempts: 5 } }, + new Error("database timeout") + ) + ).resolves.toBeUndefined(); + + expect(boundary.setStatus).toHaveBeenLastCalledWith( + expect.objectContaining({ + status: "failed", + errorCode: "USER_STATISTICS_RESET_OPERATION_FAILED", + }) + ); + }); }); diff --git a/tests/unit/lib/user-statistics-reset-service.test.ts b/tests/unit/lib/user-statistics-reset-service.test.ts index 43e9406ee..38226136a 100644 --- a/tests/unit/lib/user-statistics-reset-service.test.ts +++ b/tests/unit/lib/user-statistics-reset-service.test.ts @@ -96,11 +96,29 @@ describe("executeUserStatisticsReset", () => { keyHashes: ["key-hash"], includeActiveSessions: false, allowWhenRateLimitDisabled: true, + preserveFixed5hCostKeys: true, }); expect(sqlText(boundary.updateSet?.costResetAt)).toContain("case when"); expect(sqlText(boundary.updateSet?.limit5hCostResetAt)).toContain("case when"); }); + it("reports progress after every committed batch", async () => { + boundary.transactionResults = [{ count: 1000 }, { count: 2 }, { count: 3 }]; + boundary.executeResults = [[{ exists: false }], [{ exists: false }]]; + const progress = vi.fn().mockResolvedValue(undefined); + + await executeUserStatisticsReset( + { userId: 42, requestedAt: "2026-08-02T12:00:00.000Z" }, + progress + ); + + expect(progress.mock.calls).toEqual([ + [{ deletedMessageRequests: 1000, deletedUsageLedger: 0 }], + [{ deletedMessageRequests: 1002, deletedUsageLedger: 0 }], + [{ deletedMessageRequests: 1002, deletedUsageLedger: 3 }], + ]); + }); + it("reports deleted rows when cache cleanup fails so retries preserve progress", async () => { boundary.transactionResults = [{ count: 4 }, { count: 6 }]; boundary.executeResults = [[{ exists: false }], [{ exists: false }]]; diff --git a/tests/unit/lib/user-statistics-reset-status-store.test.ts b/tests/unit/lib/user-statistics-reset-status-store.test.ts index 7ea16b9c2..cfcd58d4b 100644 --- a/tests/unit/lib/user-statistics-reset-status-store.test.ts +++ b/tests/unit/lib/user-statistics-reset-status-store.test.ts @@ -39,6 +39,8 @@ const record = { deletedMessageRequests: 0, deletedUsageLedger: 0, errorCode: null, + fixed5hKeyIds: [9], + fixed5hPreparationVersion: null, }; describe("user statistics reset status store", () => { @@ -83,6 +85,21 @@ describe("user statistics reset status store", () => { ); }); + it("normalizes legacy records without fixed 5h key ids", async () => { + const { + fixed5hKeyIds: _fixed5hKeyIds, + fixed5hPreparationVersion: _fixed5hPreparationVersion, + ...legacyRecord + } = record; + boundary.redis.get.mockResolvedValue(JSON.stringify(legacyRecord)); + + await expect(getUserStatisticsResetStatus(record.resetId)).resolves.toEqual({ + ...legacyRecord, + fixed5hKeyIds: [], + fixed5hPreparationVersion: null, + }); + }); + it("claims one active reset and returns the existing owner on contention", async () => { boundary.redis.set.mockResolvedValueOnce("OK").mockResolvedValueOnce(null); boundary.redis.get.mockResolvedValue(record.resetId); From 09a2febd2b06c4d27ce422b664f185eee846da84 Mon Sep 17 00:00:00 2001 From: ding113 Date: Mon, 3 Aug 2026 03:42:05 +0800 Subject: [PATCH 08/18] fix(sessions): normalize browser abort errors to stable cancellation code Re-throwing the raw DOMException exposed browser-specific error text to callers. Replace it with a stable FETCH_SESSIONS_CANCELLED error so consumers can match on a known code regardless of the browser runtime. --- .../_components/active-sessions-query.test.ts | 22 +++++++++++++++++++ .../_components/active-sessions-query.ts | 4 ++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/src/app/[locale]/dashboard/sessions/_components/active-sessions-query.test.ts b/src/app/[locale]/dashboard/sessions/_components/active-sessions-query.test.ts index 6902487a8..7416e9f8c 100644 --- a/src/app/[locale]/dashboard/sessions/_components/active-sessions-query.test.ts +++ b/src/app/[locale]/dashboard/sessions/_components/active-sessions-query.test.ts @@ -61,6 +61,28 @@ describe("fetchAllSessionsPage", () => { expect(requestSignal?.aborted).toBe(true); }); + it("normalizes a rejected browser abort to the stable caller cancellation error", async () => { + api.getAllSessions.mockImplementation( + (_active: number, _inactive: number, _size: number, options: RequestInit) => + new Promise((_resolve, reject) => { + options.signal?.addEventListener("abort", () => { + reject(new DOMException("Aborted", "AbortError")); + }); + }) + ); + const controller = new AbortController(); + const request = fetchAllSessionsPage({ + activePage: 1, + inactivePage: 1, + pageSize: 20, + signal: controller.signal, + }); + + controller.abort(); + + await expect(request).rejects.toThrow("FETCH_SESSIONS_CANCELLED"); + }); + it("normalizes transport failures instead of exposing browser error text", async () => { api.getAllSessions.mockRejectedValue(new TypeError("Failed to fetch")); diff --git a/src/app/[locale]/dashboard/sessions/_components/active-sessions-query.ts b/src/app/[locale]/dashboard/sessions/_components/active-sessions-query.ts index 7e10fa976..df9d781eb 100644 --- a/src/app/[locale]/dashboard/sessions/_components/active-sessions-query.ts +++ b/src/app/[locale]/dashboard/sessions/_components/active-sessions-query.ts @@ -41,12 +41,12 @@ export async function fetchAllSessionsPage(input: { throw new Error("FETCH_SESSIONS_FAILED"); } return result.data; - } catch (cause) { + } catch { if (timeoutController.signal.aborted && !input.signal.aborted) { throw new Error("FETCH_SESSIONS_TIMEOUT"); } if (input.signal.aborted) { - throw cause; + throw new Error("FETCH_SESSIONS_CANCELLED"); } throw new Error("FETCH_SESSIONS_FAILED"); } finally { From 0a19dfb71120e93f6545e93be62f280394392407 Mon Sep 17 00:00:00 2001 From: ding113 Date: Mon, 3 Aug 2026 03:42:05 +0800 Subject: [PATCH 09/18] fix(ui): add exponential backoff and retry limit to reset polling The statistics-reset poll loop retried transient network errors indefinitely at a fixed interval. Switch to exponential backoff with a cap of five consecutive retryable failures, after which the reset is marked failed instead of leaving the UI in a perpetual loading state. --- .../_components/user/edit-user-dialog.tsx | 32 ++++++-- tests/unit/user-dialogs.test.tsx | 77 +++++++++++++++++++ 2 files changed, 102 insertions(+), 7 deletions(-) diff --git a/src/app/[locale]/dashboard/_components/user/edit-user-dialog.tsx b/src/app/[locale]/dashboard/_components/user/edit-user-dialog.tsx index 083435d1f..1b143f5d3 100644 --- a/src/app/[locale]/dashboard/_components/user/edit-user-dialog.tsx +++ b/src/app/[locale]/dashboard/_components/user/edit-user-dialog.tsx @@ -66,6 +66,10 @@ const EditUserSchema = UpdateUserSchema.extend({ type EditUserValues = z.infer; +const STATISTICS_RESET_POLL_INTERVAL_MS = 1_000; +const STATISTICS_RESET_MAX_RETRIES = 5; +const STATISTICS_RESET_RETRY_MAX_DELAY_MS = 16_000; + function buildDefaultValues(user: UserDisplay): EditUserValues { return { name: user.name || "", @@ -282,31 +286,45 @@ function EditUserDialogInner({ onOpenChange, user, onSuccess }: EditUserDialogPr if (!statisticsReset || !["queued", "running"].includes(statisticsReset.status)) return; let cancelled = false; let timer: ReturnType | undefined; + let consecutiveRetryableFailures = 0; const poll = async () => { const result = await getUserStatisticsReset(user.id, statisticsReset.resetId); if (cancelled) return; if (!result.ok) { - if (["CONNECTION_FAILED", "NETWORK_ERROR", "TIMEOUT"].includes(result.errorCode ?? "")) { - timer = setTimeout(poll, 2_000); + const retryable = ["CONNECTION_FAILED", "NETWORK_ERROR", "TIMEOUT"].includes( + result.errorCode ?? "" + ); + if (retryable && consecutiveRetryableFailures < STATISTICS_RESET_MAX_RETRIES) { + const delay = Math.min( + STATISTICS_RESET_POLL_INTERVAL_MS * 2 ** consecutiveRetryableFailures, + STATISTICS_RESET_RETRY_MAX_DELAY_MS + ); + consecutiveRetryableFailures += 1; + timer = setTimeout(poll, delay); return; } - setIsResettingAll(false); - toast.error(result.error || t("editDialog.resetData.error")); + applyStatisticsResetStatus({ + ...statisticsReset, + status: "failed", + completedAt: new Date().toISOString(), + errorCode: result.errorCode ?? "NETWORK_ERROR", + }); return; } + consecutiveRetryableFailures = 0; const next = result.data as UserStatisticsResetRecord; if (applyStatisticsResetStatus(next)) return; - timer = setTimeout(poll, 1_000); + timer = setTimeout(poll, STATISTICS_RESET_POLL_INTERVAL_MS); }; - timer = setTimeout(poll, 1_000); + timer = setTimeout(poll, STATISTICS_RESET_POLL_INTERVAL_MS); return () => { cancelled = true; if (timer) clearTimeout(timer); }; - }, [applyStatisticsResetStatus, statisticsReset, t, user.id]); + }, [applyStatisticsResetStatus, statisticsReset, user.id]); const handleResetLimitsOnly = async () => { setIsResettingLimits(true); diff --git a/tests/unit/user-dialogs.test.tsx b/tests/unit/user-dialogs.test.tsx index 199298b87..0a5f82156 100644 --- a/tests/unit/user-dialogs.test.tsx +++ b/tests/unit/user-dialogs.test.tsx @@ -45,6 +45,7 @@ const mockToggleUserEnabled = vi.fn().mockResolvedValue({ ok: true }); const mockResetUser5hLimitOnly = vi.fn().mockResolvedValue({ ok: true }); const mockResetUserLimitsOnly = vi.fn().mockResolvedValue({ ok: true }); const mockResetUserAllStatistics = vi.fn().mockResolvedValue({ ok: true }); +const mockGetUserStatisticsReset = vi.fn(); const mockAddKey = vi.fn().mockResolvedValue({ ok: true, data: { key: "sk-test-key" } }); const mockEditKey = vi.fn().mockResolvedValue({ ok: true }); const mockCreateUserOnly = vi.fn().mockResolvedValue({ ok: true, data: { user: { id: 1 } } }); @@ -58,6 +59,34 @@ vi.mock("@/actions/users", () => ({ createUserOnly: (...args: unknown[]) => mockCreateUserOnly(...args), })); +vi.mock("@/lib/api-client/v1/actions/users", () => ({ + editUser: (...args: unknown[]) => mockEditUser(...args), + getUserStatisticsReset: (...args: unknown[]) => mockGetUserStatisticsReset(...args), + removeUser: (...args: unknown[]) => mockRemoveUser(...args), + resetUserLimitsOnly: (...args: unknown[]) => mockResetUserLimitsOnly(...args), + resetUserAllStatistics: (...args: unknown[]) => mockResetUserAllStatistics(...args), + toggleUserEnabled: (...args: unknown[]) => mockToggleUserEnabled(...args), +})); + +vi.mock("@/components/ui/alert-dialog", () => { + type PropsWithChildren = { children?: ReactNode }; + const Wrap = ({ children }: PropsWithChildren) =>
{children}
; + const Button = ({ children, ...props }: PropsWithChildren & Record) => ( + + ); + return { + AlertDialog: Wrap, + AlertDialogAction: Button, + AlertDialogCancel: Button, + AlertDialogContent: Wrap, + AlertDialogDescription: Wrap, + AlertDialogFooter: Wrap, + AlertDialogHeader: Wrap, + AlertDialogTitle: Wrap, + AlertDialogTrigger: Wrap, + }; +}); + vi.mock("@/app/[locale]/dashboard/_components/user/actions/reset-user-5h-limit", () => ({ resetUser5hLimitOnly: (...args: unknown[]) => mockResetUser5hLimitOnly(...args), })); @@ -258,6 +287,20 @@ describe("EditUserDialog", () => { defaultOptions: { queries: { retry: false } }, }); vi.clearAllMocks(); + mockResetUserAllStatistics.mockResolvedValue({ + ok: true, + data: { + resetId: "00000000-0000-4000-8000-000000000001", + userId: 1, + status: "queued", + requestedAt: "2026-08-02T12:00:00.000Z", + startedAt: null, + completedAt: null, + deletedMessageRequests: 0, + deletedUsageLedger: 0, + errorCode: null, + }, + }); }); afterEach(() => { @@ -381,6 +424,40 @@ describe("EditUserDialog", () => { unmount(); }); + + test("marks a statistics reset failed after five consecutive polling retries", async () => { + vi.useFakeTimers(); + mockGetUserStatisticsReset.mockResolvedValue({ + ok: false, + error: "Service unavailable", + errorCode: "CONNECTION_FAILED", + }); + const { container, unmount } = renderWithProviders( + + ); + const buttons = Array.from(container.querySelectorAll("button")); + const resetButton = buttons.find( + (button) => + button.textContent?.trim() === messages.dashboard.userManagement.editDialog.resetData.button + ); + const confirmButton = buttons.find( + (button) => + button.textContent?.trim() === + messages.dashboard.userManagement.editDialog.resetData.confirm + ); + + act(() => resetButton?.click()); + await act(async () => confirmButton?.click()); + await act(async () => vi.advanceTimersByTimeAsync(32_000)); + + expect(mockGetUserStatisticsReset).toHaveBeenCalledTimes(6); + expect(container.querySelector('[data-testid="statistics-reset-status"]')?.textContent).toBe( + messages.dashboard.userManagement.editDialog.resetData.failed + ); + + unmount(); + vi.useRealTimers(); + }); }); describe("EditKeyDialog", () => { From 8687d07aea67f49c03030680920d3543ab3e9da3 Mon Sep 17 00:00:00 2001 From: ding113 Date: Mon, 3 Aug 2026 03:42:05 +0800 Subject: [PATCH 10/18] fix(reset-queue): guard failed-job handler and persist status safely Skip the failed-job handler when job data is missing recoverable fields so undefined resetId or userId values do not corrupt status records. Wrap status persistence in a try-catch so a transient store failure still releases the active-reset claim and propagates the original business error. Use optional chaining on job.opts for null safety. --- src/lib/user-statistics-reset/reset-queue.ts | 38 ++++++++++++------ .../lib/user-statistics-reset-queue.test.ts | 39 +++++++++++++++++++ 2 files changed, 66 insertions(+), 11 deletions(-) diff --git a/src/lib/user-statistics-reset/reset-queue.ts b/src/lib/user-statistics-reset/reset-queue.ts index 3bb1f5ca9..891599098 100644 --- a/src/lib/user-statistics-reset/reset-queue.ts +++ b/src/lib/user-statistics-reset/reset-queue.ts @@ -139,14 +139,21 @@ function getResetQueue(): Queue.Queue { }); resetQueue.process(RESET_JOB_NAME, processUserStatisticsReset); resetQueue.on("failed", async (job, error) => { + if (!job?.data?.resetId || !Number.isInteger(job.data.userId)) { + logger.error("[UserStatisticsResetQueue] job failed without recoverable data", { + error: error.message, + }); + return; + } logger.error("[UserStatisticsResetQueue] job failed", { resetId: job.data.resetId, userId: job.data.userId, attemptsMade: job.attemptsMade, error: error.message, }); - const attempts = job.opts.attempts ?? 1; - const isTerminal = job.attemptsMade >= attempts || error.message === STALLED_FAILURE_REASON; + const attempts = job.opts?.attempts ?? 1; + const isTerminal = + (job.attemptsMade ?? 0) >= attempts || error.message === STALLED_FAILURE_REASON; if (isTerminal) { try { await recordFinalFailure(job.data, error); @@ -259,15 +266,24 @@ async function processUserStatisticsReset(job: Job) errorProgress.deletedUsageLedger ), }; - await setUserStatisticsResetStatus({ - ...current, - deletedMessageRequests: baseProgress.deletedMessageRequests + progress.deletedMessageRequests, - deletedUsageLedger: baseProgress.deletedUsageLedger + progress.deletedUsageLedger, - status: isFinalAttempt ? "failed" : "queued", - startedAt, - completedAt: isFinalAttempt ? new Date().toISOString() : null, - errorCode: isFinalAttempt ? errorCode(error) : null, - }); + try { + await setUserStatisticsResetStatus({ + ...current, + deletedMessageRequests: + baseProgress.deletedMessageRequests + progress.deletedMessageRequests, + deletedUsageLedger: baseProgress.deletedUsageLedger + progress.deletedUsageLedger, + status: isFinalAttempt ? "failed" : "queued", + startedAt, + completedAt: isFinalAttempt ? new Date().toISOString() : null, + errorCode: isFinalAttempt ? errorCode(error) : null, + }); + } catch (statusError) { + logger.error("[UserStatisticsResetQueue] failed to persist attempt status", { + resetId: job.data.resetId, + userId: job.data.userId, + error: statusError instanceof Error ? statusError.message : String(statusError), + }); + } if (isFinalAttempt) { try { await releaseActiveUserStatisticsReset(job.data.userId, job.data.resetId); diff --git a/tests/unit/lib/user-statistics-reset-queue.test.ts b/tests/unit/lib/user-statistics-reset-queue.test.ts index 350365a71..1b41462e5 100644 --- a/tests/unit/lib/user-statistics-reset-queue.test.ts +++ b/tests/unit/lib/user-statistics-reset-queue.test.ts @@ -474,6 +474,45 @@ describe("user statistics reset queue", () => { ); }); + it("preserves the final business error and releases the claim when status persistence fails", async () => { + boundary.claim.mockImplementation(async (_userId: number, resetId: string) => ({ + acquired: true, + resetId, + })); + const queued = await enqueueUserStatisticsReset(42); + boundary.getStatus.mockResolvedValue({ + ...queued, + fixed5hKeyIds: [], + fixed5hPreparationVersion: 1, + }); + const resetError = new UserStatisticsResetError("USER_STATISTICS_RESET_CACHE_CLEANUP_FAILED", { + deletedMessageRequests: 5, + deletedUsageLedger: 7, + }); + boundary.execute.mockRejectedValue(resetError); + boundary.setStatus + .mockResolvedValueOnce(undefined) + .mockRejectedValueOnce(new Error("status store unavailable")); + + await expect( + boundary.processHandler?.({ data: queued, attemptsMade: 4, opts: { attempts: 5 } }) + ).rejects.toBe(resetError); + + expect(boundary.release).toHaveBeenCalledWith(42, queued.resetId); + }); + + it("ignores failed events that do not include recoverable job data", async () => { + startUserStatisticsResetQueue(); + + await expect( + boundary.failedHandler?.(undefined, new Error("job stalled more than allowable limit")) + ).resolves.toBeUndefined(); + + expect(boundary.getStatus).not.toHaveBeenCalled(); + expect(boundary.setStatus).not.toHaveBeenCalled(); + expect(boundary.release).not.toHaveBeenCalled(); + }); + it("marks max-stalled jobs failed even when attemptsMade did not advance", async () => { startUserStatisticsResetQueue(); boundary.getStatus.mockResolvedValue(existing); From 21c3e961c8579a65f2d9d6c69697b0c35ef76a2b Mon Sep 17 00:00:00 2001 From: ding113 Date: Mon, 3 Aug 2026 03:42:05 +0800 Subject: [PATCH 11/18] fix(reset-store): wait for Redis readiness before status operations getReadyRedis threw immediately when the shared client was in a connecting state, causing spurious failures during startup or reconnect. Make it async and await the ready or end event with a configurable timeout so callers tolerate transient connection states. --- .../reset-status-store.ts | 41 +++++++++++++---- ...user-statistics-reset-status-store.test.ts | 45 +++++++++++++++++-- 2 files changed, 75 insertions(+), 11 deletions(-) diff --git a/src/lib/user-statistics-reset/reset-status-store.ts b/src/lib/user-statistics-reset/reset-status-store.ts index 4dc10ebb8..ee189ec02 100644 --- a/src/lib/user-statistics-reset/reset-status-store.ts +++ b/src/lib/user-statistics-reset/reset-status-store.ts @@ -1,6 +1,7 @@ import "server-only"; import type Redis from "ioredis"; +import { getEnvConfig } from "@/lib/config/env.schema"; import { getRedisClient } from "@/lib/redis/client"; import { RedisKVStore } from "@/lib/redis/redis-kv-store"; import type { UserStatisticsResetStoredRecord } from "./types"; @@ -13,7 +14,7 @@ const statusStore = new RedisKVStore({ defaultTtlSeconds: RESET_STATUS_TTL_SECONDS, }); -type ResetRedis = Pick & { +type ResetRedis = Pick & { eval(...args: [script: string, numkeys: number, ...keysAndArgs: string[]]): Promise; }; @@ -23,17 +24,39 @@ if redis.call('GET', KEYS[1]) == ARGV[1] then end return 0`; -function getReadyRedis(): ResetRedis { +async function getReadyRedis(): Promise { const redis = getRedisClient({ allowWhenRateLimitDisabled: true }) as ResetRedis | null; - if (redis?.status !== "ready") { + if (!redis || redis.status === "end") { throw new Error("USER_STATISTICS_RESET_REDIS_UNAVAILABLE"); } - return redis; + if (redis.status === "ready") return redis; + + return new Promise((resolve, reject) => { + let settled = false; + const finish = (result: { redis: ResetRedis } | { error: Error }) => { + if (settled) return; + settled = true; + clearTimeout(timeoutId); + redis.removeListener("ready", onReady); + redis.removeListener("end", onEnd); + if ("redis" in result) resolve(result.redis); + else reject(result.error); + }; + const onReady = () => finish({ redis }); + const onEnd = () => finish({ error: new Error("USER_STATISTICS_RESET_REDIS_UNAVAILABLE") }); + const timeoutId = setTimeout(onEnd, getEnvConfig().REDIS_COMMAND_TIMEOUT_MS); + + redis.once("ready", onReady); + redis.once("end", onEnd); + if (redis.status === "ready") onReady(); + else if (redis.status === "end") onEnd(); + }); } export async function setUserStatisticsResetStatus( record: UserStatisticsResetStoredRecord ): Promise { + await getReadyRedis(); if (!(await statusStore.set(record.resetId, record))) { throw new Error("USER_STATISTICS_RESET_STATUS_WRITE_FAILED"); } @@ -42,7 +65,8 @@ export async function setUserStatisticsResetStatus( export async function getUserStatisticsResetStatus( resetId: string ): Promise { - const raw = await getReadyRedis().get(`${RESET_STATUS_PREFIX}${resetId}`); + const redis = await getReadyRedis(); + const raw = await redis.get(`${RESET_STATUS_PREFIX}${resetId}`); if (!raw) return null; try { const record = JSON.parse(raw) as UserStatisticsResetStoredRecord; @@ -57,14 +81,15 @@ export async function getUserStatisticsResetStatus( } export async function deleteUserStatisticsResetStatus(resetId: string): Promise { - await getReadyRedis().del(`${RESET_STATUS_PREFIX}${resetId}`); + const redis = await getReadyRedis(); + await redis.del(`${RESET_STATUS_PREFIX}${resetId}`); } export async function claimActiveUserStatisticsReset( userId: number, resetId: string ): Promise<{ acquired: boolean; resetId: string }> { - const redis = getReadyRedis(); + const redis = await getReadyRedis(); const key = `${ACTIVE_RESET_PREFIX}${userId}`; const result = await redis.set(key, resetId, "EX", RESET_STATUS_TTL_SECONDS, "NX"); if (result === "OK") { @@ -82,6 +107,6 @@ export async function releaseActiveUserStatisticsReset( userId: number, resetId: string ): Promise { - const redis = getReadyRedis(); + const redis = await getReadyRedis(); await redis.eval(LUA_COMPARE_DELETE, 1, `${ACTIVE_RESET_PREFIX}${userId}`, resetId); } diff --git a/tests/unit/lib/user-statistics-reset-status-store.test.ts b/tests/unit/lib/user-statistics-reset-status-store.test.ts index cfcd58d4b..5b2935cb5 100644 --- a/tests/unit/lib/user-statistics-reset-status-store.test.ts +++ b/tests/unit/lib/user-statistics-reset-status-store.test.ts @@ -1,16 +1,25 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; const boundary = vi.hoisted(() => ({ + listeners: new Map void>>(), redis: { status: "ready", get: vi.fn(), set: vi.fn(), del: vi.fn(), eval: vi.fn(), + once: vi.fn(), + removeListener: vi.fn(), }, storeSet: vi.fn(), })); +function emitRedis(event: string, ...args: unknown[]) { + const listeners = [...(boundary.listeners.get(event) ?? [])]; + boundary.listeners.delete(event); + for (const listener of listeners) listener(...args); +} + vi.mock("@/lib/redis/client", () => ({ getRedisClient: () => boundary.redis, })); @@ -46,15 +55,32 @@ const record = { describe("user statistics reset status store", () => { beforeEach(() => { boundary.redis.status = "ready"; + boundary.listeners.clear(); for (const mock of [ boundary.redis.get, boundary.redis.set, boundary.redis.del, boundary.redis.eval, + boundary.redis.once, + boundary.redis.removeListener, boundary.storeSet, ]) { mock.mockReset(); } + boundary.redis.once.mockImplementation( + (event: string, listener: (...args: unknown[]) => void) => { + const listeners = boundary.listeners.get(event) ?? new Set(); + listeners.add(listener); + boundary.listeners.set(event, listeners); + return boundary.redis; + } + ); + boundary.redis.removeListener.mockImplementation( + (event: string, listener: (...args: unknown[]) => void) => { + boundary.listeners.get(event)?.delete(listener); + return boundary.redis; + } + ); boundary.storeSet.mockResolvedValue(true); }); @@ -78,11 +104,24 @@ describe("user statistics reset status store", () => { await expect(getUserStatisticsResetStatus(record.resetId)).rejects.toThrow( "USER_STATISTICS_RESET_STATUS_INVALID" ); + }); + it("waits for the shared Redis client to become ready before reading and writing", async () => { boundary.redis.status = "connecting"; - await expect(getUserStatisticsResetStatus(record.resetId)).rejects.toThrow( - "USER_STATISTICS_RESET_REDIS_UNAVAILABLE" - ); + boundary.redis.get.mockResolvedValue(JSON.stringify(record)); + + const read = getUserStatisticsResetStatus(record.resetId); + const write = setUserStatisticsResetStatus(record); + expect(boundary.redis.get).not.toHaveBeenCalled(); + expect(boundary.storeSet).not.toHaveBeenCalled(); + + boundary.redis.status = "ready"; + emitRedis("ready"); + + await expect(read).resolves.toEqual(record); + await expect(write).resolves.toBeUndefined(); + expect(boundary.redis.get).toHaveBeenCalledOnce(); + expect(boundary.storeSet).toHaveBeenCalledWith(record.resetId, record); }); it("normalizes legacy records without fixed 5h key ids", async () => { From de0f56d94d74ed4c98b0eec650545c11d2c74c89 Mon Sep 17 00:00:00 2001 From: ding113 Date: Mon, 3 Aug 2026 03:42:05 +0800 Subject: [PATCH 12/18] fix(cost-cache): count scan failures in pipeline error results When the Redis pipeline threw after some scan calls had already failed, the returned errorCount omitted those earlier scan failures. Include the accumulated scan error count so callers see the true failure total. --- src/lib/redis/cost-cache-cleanup.ts | 1 + .../unit/lib/redis/cost-cache-cleanup.test.ts | 23 +++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/src/lib/redis/cost-cache-cleanup.ts b/src/lib/redis/cost-cache-cleanup.ts index 6a13d5451..bd003f999 100644 --- a/src/lib/redis/cost-cache-cleanup.ts +++ b/src/lib/redis/cost-cache-cleanup.ts @@ -228,6 +228,7 @@ export async function clearUserCostCache( activeSessionsDeleted, durationMs: Date.now() - startTime, cleanupFailed: true, + errorCount: scanErrorCount + 1, }; } diff --git a/tests/unit/lib/redis/cost-cache-cleanup.test.ts b/tests/unit/lib/redis/cost-cache-cleanup.test.ts index c046ea1a8..bf1fee9c0 100644 --- a/tests/unit/lib/redis/cost-cache-cleanup.test.ts +++ b/tests/unit/lib/redis/cost-cache-cleanup.test.ts @@ -312,6 +312,29 @@ describe("clearUserCostCache", () => { ); }); + test("pipeline exceptions include scan failures in the returned error count", async () => { + let scanCall = 0; + scanPatternMock.mockImplementation(async () => { + scanCall += 1; + if (scanCall === 1) throw new Error("scan failed"); + if (scanCall === 2) return ["key:1:cost_daily"]; + return []; + }); + redisPipelineMock.exec.mockRejectedValue(new Error("Connection reset")); + + const { clearUserCostCache } = await import("@/lib/redis/cost-cache-cleanup"); + const result = await clearUserCostCache({ + userId: 10, + keyIds: [1], + keyHashes: [], + }); + + expect(result).toMatchObject({ + cleanupFailed: true, + errorCount: 2, + }); + }); + test("no keys (empty keyIds/keyHashes) -- only user patterns scanned", async () => { scanPatternMock.mockResolvedValue([]); From 9c91d45cfbbcdb134040a3f3db1f1aa5ef39a38c Mon Sep 17 00:00:00 2001 From: ding113 Date: Mon, 3 Aug 2026 03:42:05 +0800 Subject: [PATCH 13/18] fix(api): use standard public detail for missing statistics reset The 404 response for a not-found statistics reset leaked an implementation-specific detail string. Use the shared publicActionErrorDetail helper so the message matches other 404 responses across the API surface. --- src/app/api/v1/resources/users/handlers.ts | 5 +++-- tests/api/v1/users/users.test.ts | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/app/api/v1/resources/users/handlers.ts b/src/app/api/v1/resources/users/handlers.ts index ef6052786..75494aca8 100644 --- a/src/app/api/v1/resources/users/handlers.ts +++ b/src/app/api/v1/resources/users/handlers.ts @@ -25,6 +25,7 @@ import { UsersUsageBatchSchema, UserUpdateSchema, } from "@/lib/api/v1/schemas/users"; +import type { UserStatisticsResetRecord } from "@/lib/user-statistics-reset/types"; export async function listUsers(c: Context): Promise { const query = UserListQuerySchema.safeParse({ @@ -239,7 +240,7 @@ export async function getUserStatisticsReset(c: Context): Promise { if (!params.success) return fromZodError(params.error, new URL(c.req.url).pathname); const { findUserStatisticsReset } = await import("@/lib/user-statistics-reset/reset-queue"); - let reset; + let reset: UserStatisticsResetRecord | null; try { reset = await findUserStatisticsReset(params.data.id, params.data.resetId); } catch { @@ -255,7 +256,7 @@ export async function getUserStatisticsReset(c: Context): Promise { status: 404, instance: new URL(c.req.url).pathname, errorCode: "user.statistics_reset_not_found", - detail: "Statistics reset not found.", + detail: publicActionErrorDetail(404), }); } return jsonResponse(reset); diff --git a/tests/api/v1/users/users.test.ts b/tests/api/v1/users/users.test.ts index 792300c7a..3eab3be4f 100644 --- a/tests/api/v1/users/users.test.ts +++ b/tests/api/v1/users/users.test.ts @@ -390,6 +390,22 @@ describe("v1 users endpoints", () => { expect(resetStatus.json).toMatchObject({ status: "running", deletedMessageRequests: 1000 }); }); + test("uses the standard public detail when a statistics reset is not found", async () => { + findUserStatisticsResetMock.mockResolvedValueOnce(null); + + const result = await callV1Route({ + method: "GET", + pathname: "/api/v1/users/1/statistics-resets/00000000-0000-4000-8000-000000000001", + headers, + }); + + expect(result.response.status).toBe(404); + expect(result.json).toMatchObject({ + errorCode: "user.statistics_reset_not_found", + detail: "Not found", + }); + }); + test("maps structured authorization action errors to HTTP status codes", async () => { getUserLimitUsageMock.mockResolvedValueOnce({ ok: false, From a7019a687c3a113b47dd9ef391a0d5c3b463d964 Mon Sep 17 00:00:00 2001 From: ding113 Date: Mon, 3 Aug 2026 04:04:41 +0800 Subject: [PATCH 14/18] fix(dashboard): harden statistics reset polling and dev queue safety Replace the old behaviour of marking a reset as failed when polling retries were exhausted: the authoritative server status is now preserved and a recoverable error banner with a retry button is shown instead. Each poll request is given a 15 s AbortSignal timeout so a hung request no longer blocks subsequent retries. Guard prepareUserStatisticsResetFixed5h against invalid or missing Redis cutoff values by returning null instead of constructing an epoch date. Throw early when getResetQueue is called in development to prevent accidental Bull/Redis processor creation outside production. Add i18n strings for statusUnavailable and retryStatus across all supported locales. --- messages/en/dashboard.json | 2 + messages/ja/dashboard.json | 2 + messages/ru/dashboard.json | 2 + messages/zh-CN/dashboard.json | 2 + messages/zh-TW/dashboard.json | 2 + .../_components/user/edit-user-dialog.tsx | 58 +++++++++++++++---- src/lib/api-client/v1/actions/users.ts | 11 ++-- src/lib/redis/cost-cache-cleanup.ts | 10 +++- src/lib/user-statistics-reset/reset-queue.ts | 3 + tests/unit/api/v1/api-client-actions.test.ts | 11 ++++ .../unit/lib/redis/cost-cache-cleanup.test.ts | 11 ++++ .../lib/user-statistics-reset-queue.test.ts | 20 ++++++- tests/unit/user-dialogs.test.tsx | 58 ++++++++++++++++++- 13 files changed, 173 insertions(+), 19 deletions(-) diff --git a/messages/en/dashboard.json b/messages/en/dashboard.json index 358a19e2c..5ed3ad430 100644 --- a/messages/en/dashboard.json +++ b/messages/en/dashboard.json @@ -1903,6 +1903,8 @@ "running": "Reset in progress", "completed": "Reset completed", "failed": "Reset failed. You can retry.", + "statusUnavailable": "Unable to refresh the reset status", + "retryStatus": "Retry status", "success": "All statistics have been reset" } }, diff --git a/messages/ja/dashboard.json b/messages/ja/dashboard.json index ee16d89c7..5bfb26738 100644 --- a/messages/ja/dashboard.json +++ b/messages/ja/dashboard.json @@ -1881,6 +1881,8 @@ "running": "統計をリセットしています", "completed": "リセットが完了しました", "failed": "リセットに失敗しました。再試行できます。", + "statusUnavailable": "リセット状態を更新できません", + "retryStatus": "状態を再取得", "success": "すべての統計がリセットされました" } }, diff --git a/messages/ru/dashboard.json b/messages/ru/dashboard.json index cbd7a6f3d..453a0e427 100644 --- a/messages/ru/dashboard.json +++ b/messages/ru/dashboard.json @@ -1886,6 +1886,8 @@ "running": "Статистика сбрасывается", "completed": "Сброс завершен", "failed": "Сброс не выполнен. Можно повторить.", + "statusUnavailable": "Не удалось обновить статус сброса", + "retryStatus": "Повторить проверку", "success": "Вся статистика сброшена" } }, diff --git a/messages/zh-CN/dashboard.json b/messages/zh-CN/dashboard.json index 4578e630b..9b1d50786 100644 --- a/messages/zh-CN/dashboard.json +++ b/messages/zh-CN/dashboard.json @@ -1904,6 +1904,8 @@ "running": "正在重置统计", "completed": "重置已完成", "failed": "重置失败，可以重试", + "statusUnavailable": "暂时无法更新重置状态", + "retryStatus": "重试状态查询", "success": "所有统计已重置" } }, diff --git a/messages/zh-TW/dashboard.json b/messages/zh-TW/dashboard.json index 83a589fa5..112d7afde 100644 --- a/messages/zh-TW/dashboard.json +++ b/messages/zh-TW/dashboard.json @@ -1889,6 +1889,8 @@ "running": "正在重設統計", "completed": "重設已完成", "failed": "重設失敗，可以重試", + "statusUnavailable": "暫時無法更新重設狀態", + "retryStatus": "重試狀態查詢", "success": "所有統計已重置" } }, diff --git a/src/app/[locale]/dashboard/_components/user/edit-user-dialog.tsx b/src/app/[locale]/dashboard/_components/user/edit-user-dialog.tsx index 1b143f5d3..7791e093d 100644 --- a/src/app/[locale]/dashboard/_components/user/edit-user-dialog.tsx +++ b/src/app/[locale]/dashboard/_components/user/edit-user-dialog.tsx @@ -1,7 +1,7 @@ "use client"; import { useQueryClient } from "@tanstack/react-query"; -import { Loader2, RotateCcw, Trash2, UserCog } from "lucide-react"; +import { Loader2, RefreshCw, RotateCcw, Trash2, UserCog } from "lucide-react"; import { useRouter } from "next/navigation"; import { useLocale, useTranslations } from "next-intl"; import { useCallback, useEffect, useMemo, useState, useTransition } from "react"; @@ -67,6 +67,7 @@ const EditUserSchema = UpdateUserSchema.extend({ type EditUserValues = z.infer; const STATISTICS_RESET_POLL_INTERVAL_MS = 1_000; +const STATISTICS_RESET_REQUEST_TIMEOUT_MS = 15_000; const STATISTICS_RESET_MAX_RETRIES = 5; const STATISTICS_RESET_RETRY_MAX_DELAY_MS = 16_000; @@ -103,6 +104,7 @@ function EditUserDialogInner({ onOpenChange, user, onSuccess }: EditUserDialogPr const [isResettingAll, setIsResettingAll] = useState(false); const [resetAllDialogOpen, setResetAllDialogOpen] = useState(false); const [statisticsReset, setStatisticsReset] = useState(null); + const [statisticsResetPollFailed, setStatisticsResetPollFailed] = useState(false); const [isResetting5h, setIsResetting5h] = useState(false); const [reset5hDialogOpen, setReset5hDialogOpen] = useState(false); const [isResettingLimits, setIsResettingLimits] = useState(false); @@ -266,6 +268,7 @@ function EditUserDialogInner({ onOpenChange, user, onSuccess }: EditUserDialogPr const handleResetAllStatistics = async () => { setIsResettingAll(true); + setStatisticsResetPollFailed(false); try { const res = await resetUserAllStatistics(user.id); if (!res.ok) { @@ -283,17 +286,34 @@ function EditUserDialogInner({ onOpenChange, user, onSuccess }: EditUserDialogPr }; useEffect(() => { - if (!statisticsReset || !["queued", "running"].includes(statisticsReset.status)) return; + if ( + !statisticsReset || + statisticsResetPollFailed || + !["queued", "running"].includes(statisticsReset.status) + ) + return; let cancelled = false; let timer: ReturnType | undefined; + let activeRequestController: AbortController | undefined; let consecutiveRetryableFailures = 0; const poll = async () => { - const result = await getUserStatisticsReset(user.id, statisticsReset.resetId); + const requestController = new AbortController(); + activeRequestController = requestController; + const requestTimeout = window.setTimeout( + () => requestController.abort(), + STATISTICS_RESET_REQUEST_TIMEOUT_MS + ); + const result = await getUserStatisticsReset(user.id, statisticsReset.resetId, { + signal: requestController.signal, + }); + window.clearTimeout(requestTimeout); + if (activeRequestController === requestController) activeRequestController = undefined; if (cancelled) return; if (!result.ok) { + const resultErrorCode = requestController.signal.aborted ? "TIMEOUT" : result.errorCode; const retryable = ["CONNECTION_FAILED", "NETWORK_ERROR", "TIMEOUT"].includes( - result.errorCode ?? "" + resultErrorCode ?? "" ); if (retryable && consecutiveRetryableFailures < STATISTICS_RESET_MAX_RETRIES) { const delay = Math.min( @@ -304,16 +324,12 @@ function EditUserDialogInner({ onOpenChange, user, onSuccess }: EditUserDialogPr timer = setTimeout(poll, delay); return; } - applyStatisticsResetStatus({ - ...statisticsReset, - status: "failed", - completedAt: new Date().toISOString(), - errorCode: result.errorCode ?? "NETWORK_ERROR", - }); + setStatisticsResetPollFailed(true); return; } consecutiveRetryableFailures = 0; + setStatisticsResetPollFailed(false); const next = result.data as UserStatisticsResetRecord; if (applyStatisticsResetStatus(next)) return; timer = setTimeout(poll, STATISTICS_RESET_POLL_INTERVAL_MS); @@ -323,8 +339,9 @@ function EditUserDialogInner({ onOpenChange, user, onSuccess }: EditUserDialogPr return () => { cancelled = true; if (timer) clearTimeout(timer); + activeRequestController?.abort(); }; - }, [applyStatisticsResetStatus, statisticsReset, user.id]); + }, [applyStatisticsResetStatus, statisticsReset, statisticsResetPollFailed, user.id]); const handleResetLimitsOnly = async () => { setIsResettingLimits(true); @@ -563,6 +580,25 @@ function EditUserDialogInner({ onOpenChange, user, onSuccess }: EditUserDialogPr {t(`editDialog.resetData.${statisticsReset.status}`)}

) : null} + {statisticsResetPollFailed ? ( +
+

+ {t("editDialog.resetData.statusUnavailable")} +

+ +
+ ) : null} diff --git a/src/lib/api-client/v1/actions/users.ts b/src/lib/api-client/v1/actions/users.ts index 9b2a53f0d..5d3a81529 100644 --- a/src/lib/api-client/v1/actions/users.ts +++ b/src/lib/api-client/v1/actions/users.ts @@ -167,10 +167,13 @@ export function resetUserAllStatistics(userId: number) { return toActionResult(apiPost(`/api/v1/users/${userId}/statistics:reset`)); } -export async function getUserStatisticsReset(userId: number, resetId: string) { - const result = await toActionResult( - apiGet(`/api/v1/users/${userId}/statistics-resets/${encodeURIComponent(resetId)}`) - ); +export async function getUserStatisticsReset( + userId: number, + resetId: string, + options?: { signal?: AbortSignal } +) { + const path = `/api/v1/users/${userId}/statistics-resets/${encodeURIComponent(resetId)}`; + const result = await toActionResult(options ? apiGet(path, options) : apiGet(path)); return !result.ok && !result.errorCode ? { ...result, errorCode: "NETWORK_ERROR" } : result; } diff --git a/src/lib/redis/cost-cache-cleanup.ts b/src/lib/redis/cost-cache-cleanup.ts index bd003f999..d054cf198 100644 --- a/src/lib/redis/cost-cache-cleanup.ts +++ b/src/lib/redis/cost-cache-cleanup.ts @@ -80,7 +80,15 @@ export async function prepareUserStatisticsResetFixed5h(input: { ...keys, STATISTICS_RESET_PREPARE_TTL_SECONDS ); - const cutoff = new Date(Number(cutoffMilliseconds)); + if ( + (typeof cutoffMilliseconds !== "number" && typeof cutoffMilliseconds !== "string") || + cutoffMilliseconds === "" + ) { + return null; + } + const cutoffValue = Number(cutoffMilliseconds); + if (!Number.isFinite(cutoffValue) || cutoffValue <= 0) return null; + const cutoff = new Date(cutoffValue); return Number.isFinite(cutoff.getTime()) ? cutoff.toISOString() : null; } diff --git a/src/lib/user-statistics-reset/reset-queue.ts b/src/lib/user-statistics-reset/reset-queue.ts index 891599098..cde10a70f 100644 --- a/src/lib/user-statistics-reset/reset-queue.ts +++ b/src/lib/user-statistics-reset/reset-queue.ts @@ -122,6 +122,9 @@ async function ensurePreparedReset( } function getResetQueue(): Queue.Queue { + if (process.env.NODE_ENV === "development") { + throw new Error("USER_STATISTICS_RESET_QUEUE_DISABLED_IN_DEVELOPMENT"); + } if (resetQueue) return resetQueue; const redisUrl = process.env.REDIS_URL; if (!redisUrl) { diff --git a/tests/unit/api/v1/api-client-actions.test.ts b/tests/unit/api/v1/api-client-actions.test.ts index b48f1de8e..36307abee 100644 --- a/tests/unit/api/v1/api-client-actions.test.ts +++ b/tests/unit/api/v1/api-client-actions.test.ts @@ -89,6 +89,17 @@ describe("v1 action compatibility client", () => { }); }); + test("passes an AbortSignal to statistics reset polling", async () => { + getMock.mockResolvedValue({ status: "running" }); + const controller = new AbortController(); + + await users.getUserStatisticsReset(42, "reset-id", { signal: controller.signal }); + + expect(getMock).toHaveBeenCalledWith("/api/v1/users/42/statistics-resets/reset-id", { + signal: controller.signal, + }); + }); + test("preserves the physical request locator for every Session payload endpoint", async () => { getMock.mockResolvedValue({ exists: true, response: "ok" }); diff --git a/tests/unit/lib/redis/cost-cache-cleanup.test.ts b/tests/unit/lib/redis/cost-cache-cleanup.test.ts index bf1fee9c0..2e9663a1e 100644 --- a/tests/unit/lib/redis/cost-cache-cleanup.test.ts +++ b/tests/unit/lib/redis/cost-cache-cleanup.test.ts @@ -158,6 +158,17 @@ describe("clearUserCostCache", () => { ); }); + test("rejects invalid fixed 5h cutoff values returned by Redis", async () => { + const { prepareUserStatisticsResetFixed5h } = await import("@/lib/redis/cost-cache-cleanup"); + + for (const invalid of [null, undefined, "", 0, "not-a-timestamp", []]) { + redisMock.eval.mockResolvedValueOnce(invalid); + await expect( + prepareUserStatisticsResetFixed5h({ resetId: "reset-1", userId: 10, keyIds: [] }) + ).resolves.toBeNull(); + } + }); + test("returns metrics (costKeysDeleted, activeSessionsDeleted, durationMs)", async () => { scanPatternMock.mockImplementation(async (_redis: unknown, pattern: string) => { if (pattern === "key:1:cost_*") return ["key:1:cost_daily"]; diff --git a/tests/unit/lib/user-statistics-reset-queue.test.ts b/tests/unit/lib/user-statistics-reset-queue.test.ts index 1b41462e5..ae4886b8d 100644 --- a/tests/unit/lib/user-statistics-reset-queue.test.ts +++ b/tests/unit/lib/user-statistics-reset-queue.test.ts @@ -1,4 +1,4 @@ -import { beforeEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; const boundary = vi.hoisted(() => ({ processHandler: null as null | ((job: any) => Promise), @@ -83,6 +83,8 @@ const existing = { }; describe("user statistics reset queue", () => { + const originalNodeEnv = process.env.NODE_ENV; + beforeEach(async () => { await stopUserStatisticsResetQueue(); process.env.REDIS_URL = "redis://localhost:6379"; @@ -116,6 +118,10 @@ describe("user statistics reset queue", () => { boundary.findKeyIds.mockResolvedValue([9]); }); + afterEach(() => { + process.env.NODE_ENV = originalNodeEnv; + }); + it("returns the existing active reset instead of enqueueing a competitor", async () => { boundary.claim.mockResolvedValue({ acquired: false, resetId: existing.resetId }); boundary.getStatus.mockResolvedValue(existing); @@ -320,6 +326,18 @@ describe("user statistics reset queue", () => { expect(boundary.processHandler).toBeNull(); }); + it("does not construct a Bull processor on demand in development", async () => { + process.env.NODE_ENV = "development"; + + await expect(enqueueUserStatisticsReset(42)).rejects.toThrow( + "USER_STATISTICS_RESET_QUEUE_DISABLED_IN_DEVELOPMENT" + ); + + expect(boundary.processHandler).toBeNull(); + expect(boundary.add).not.toHaveBeenCalled(); + expect(boundary.setStatus).not.toHaveBeenCalled(); + }); + it("moves a Bull job through running to completed and releases the active claim", async () => { boundary.claim.mockImplementation(async (_userId: number, resetId: string) => ({ acquired: true, diff --git a/tests/unit/user-dialogs.test.tsx b/tests/unit/user-dialogs.test.tsx index 0a5f82156..16e521898 100644 --- a/tests/unit/user-dialogs.test.tsx +++ b/tests/unit/user-dialogs.test.tsx @@ -425,7 +425,7 @@ describe("EditUserDialog", () => { unmount(); }); - test("marks a statistics reset failed after five consecutive polling retries", async () => { + test("keeps the authoritative reset state after polling retries are exhausted", async () => { vi.useFakeTimers(); mockGetUserStatisticsReset.mockResolvedValue({ ok: false, @@ -452,10 +452,64 @@ describe("EditUserDialog", () => { expect(mockGetUserStatisticsReset).toHaveBeenCalledTimes(6); expect(container.querySelector('[data-testid="statistics-reset-status"]')?.textContent).toBe( - messages.dashboard.userManagement.editDialog.resetData.failed + messages.dashboard.userManagement.editDialog.resetData.queued ); + expect( + container.querySelector('[data-testid="statistics-reset-poll-error"]')?.textContent + ).toBe(messages.dashboard.userManagement.editDialog.resetData.statusUnavailable); + expect(mockResetUserAllStatistics).toHaveBeenCalledTimes(1); + + const retryStatusButton = Array.from(container.querySelectorAll("button")).find( + (button) => + button.textContent?.trim() === + messages.dashboard.userManagement.editDialog.resetData.retryStatus + ); + act(() => retryStatusButton?.click()); + await act(async () => vi.advanceTimersByTimeAsync(1_000)); + + expect(mockGetUserStatisticsReset).toHaveBeenCalledTimes(7); + expect(mockResetUserAllStatistics).toHaveBeenCalledTimes(1); + + unmount(); + vi.useRealTimers(); + }); + + test("aborts a hung statistics reset status request and continues polling", async () => { + vi.useFakeTimers(); + const requestSignals: AbortSignal[] = []; + mockGetUserStatisticsReset.mockImplementation( + (_userId: number, _resetId: string, options: { signal: AbortSignal }) => { + requestSignals.push(options.signal); + return new Promise((resolve) => { + options.signal.addEventListener("abort", () => { + resolve({ ok: false, error: "aborted", errorCode: "NETWORK_ERROR" }); + }); + }); + } + ); + const { container, unmount } = renderWithProviders( + + ); + const buttons = Array.from(container.querySelectorAll("button")); + const resetButton = buttons.find( + (button) => + button.textContent?.trim() === messages.dashboard.userManagement.editDialog.resetData.button + ); + const confirmButton = buttons.find( + (button) => + button.textContent?.trim() === + messages.dashboard.userManagement.editDialog.resetData.confirm + ); + + act(() => resetButton?.click()); + await act(async () => confirmButton?.click()); + await act(async () => vi.advanceTimersByTimeAsync(17_000)); + + expect(requestSignals[0]?.aborted).toBe(true); + expect(mockGetUserStatisticsReset).toHaveBeenCalledTimes(2); unmount(); + expect(requestSignals[1]?.aborted).toBe(true); vi.useRealTimers(); }); }); From 406c83b10f466134f017f79d395f9d696b392252 Mon Sep 17 00:00:00 2001 From: ding113 Date: Mon, 3 Aug 2026 04:22:58 +0800 Subject: [PATCH 15/18] fix(reset): delegate Redis readiness check to the reset queue resetUserAllStatistics previously rejected requests with a cold/unready Redis client when fixed 5h cost windows were active, surfacing a CONNECTION_FAILED error before the job was even enqueued. The reset queue itself already handles Redis readiness, so this pre-enqueue guard was redundant and blocked otherwise valid reset requests during cold-start windows. The guard and its associated fixed-5h detection logic are removed; the corresponding test now verifies that enqueue proceeds and the queue receives the request without the action layer checking Redis status. --- src/actions/users.ts | 17 ----------------- .../actions/users-reset-all-statistics.test.ts | 10 +++++----- 2 files changed, 5 insertions(+), 22 deletions(-) diff --git a/src/actions/users.ts b/src/actions/users.ts index 58c05a28e..e8b05102e 100644 --- a/src/actions/users.ts +++ b/src/actions/users.ts @@ -2366,23 +2366,6 @@ export async function resetUserAllStatistics( } const keys = await findKeyList(userId); - const requiresRedisForFixed5h = - ((user.limit5hUsd ?? 0) > 0 && (user.limit5hResetMode ?? "rolling") === "fixed") || - keys.some( - (key) => (key.limit5hUsd ?? 0) > 0 && (key.limit5hResetMode ?? "rolling") === "fixed" - ); - - if (requiresRedisForFixed5h) { - const redis = getRedisClient({ allowWhenRateLimitDisabled: true }); - if (redis?.status !== "ready") { - return { - ok: false, - error: tError("CONNECTION_FAILED"), - errorCode: ERROR_CODES.CONNECTION_FAILED, - }; - } - } - const { enqueueUserStatisticsReset } = await import("@/lib/user-statistics-reset/reset-queue"); enqueueStarted = true; const reset = await enqueueUserStatisticsReset(userId, { diff --git a/tests/unit/actions/users-reset-all-statistics.test.ts b/tests/unit/actions/users-reset-all-statistics.test.ts index 3c38a6d61..616cca122 100644 --- a/tests/unit/actions/users-reset-all-statistics.test.ts +++ b/tests/unit/actions/users-reset-all-statistics.test.ts @@ -73,20 +73,20 @@ describe("resetUserAllStatistics", () => { expect(result).toMatchObject({ ok: false, errorCode: ERROR_CODES.NOT_FOUND }); }); - test("keeps the fixed 5h Redis availability guard before enqueue", async () => { + test("delegates cold Redis readiness handling to the reset queue", async () => { mocks.findUserById.mockResolvedValue({ id: 123, limit5hUsd: 10, limit5hResetMode: "fixed", }); - mocks.getRedisClient.mockReturnValue(null); + mocks.getRedisClient.mockReturnValue({ status: "connecting" }); const { resetUserAllStatistics } = await import("@/actions/users"); const result = await resetUserAllStatistics(123); - expect(result).toMatchObject({ ok: false, errorCode: ERROR_CODES.CONNECTION_FAILED }); - expect(mocks.getRedisClient).toHaveBeenCalledWith({ allowWhenRateLimitDisabled: true }); - expect(mocks.enqueue).not.toHaveBeenCalled(); + expect(result).toEqual({ ok: true, data: queuedReset }); + expect(mocks.getRedisClient).not.toHaveBeenCalled(); + expect(mocks.enqueue).toHaveBeenCalledWith(123, { fixed5hKeyIds: [] }); }); test("queues the reset and returns its durable status", async () => { From adf93c5539fe259b5fc8811a8c0c4c5bf018c2ab Mon Sep 17 00:00:00 2001 From: ding113 Date: Mon, 3 Aug 2026 04:40:14 +0800 Subject: [PATCH 16/18] test(reset): align fixed 5h reset tests with queue-owned Redis readiness Replace assertions that expected resetUserAllStatistics to fail with CONNECTION_FAILED when Redis is unavailable. The reset now succeeds and delegates fixed 5h readiness to the background queue via enqueueUserStatisticsReset, so Redis is never contacted directly during the action call. --- tests/unit/actions/users-reset-5h-only.test.ts | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/tests/unit/actions/users-reset-5h-only.test.ts b/tests/unit/actions/users-reset-5h-only.test.ts index b451f8e76..ac366abc4 100644 --- a/tests/unit/actions/users-reset-5h-only.test.ts +++ b/tests/unit/actions/users-reset-5h-only.test.ts @@ -329,7 +329,7 @@ describe("full reset compatibility with user 5h marker", () => { expect(invalidateCachedUserMock).not.toHaveBeenCalled(); }); - test("full statistics reset fails when fixed 5h state exists but Redis is unavailable", async () => { + test("full statistics reset delegates fixed user 5h readiness to the background queue", async () => { findUserByIdMock.mockResolvedValue({ id: 123, name: "Test User", @@ -342,12 +342,15 @@ describe("full reset compatibility with user 5h marker", () => { const { resetUserAllStatistics } = await import("@/actions/users"); const result = await resetUserAllStatistics(123); - expect(result.ok).toBe(false); - expect(result.errorCode).toBe(ERROR_CODES.CONNECTION_FAILED); + expect(result.ok).toBe(true); + expect(getRedisClientMock).not.toHaveBeenCalled(); + expect(enqueueUserStatisticsResetMock).toHaveBeenCalledWith(123, { + fixed5hKeyIds: [], + }); expect(txUpdateSetMock).not.toHaveBeenCalled(); }); - test("full statistics reset fails when a child key has fixed 5h state and Redis is unavailable", async () => { + test("full statistics reset delegates fixed child-key 5h readiness to the background queue", async () => { findUserByIdMock.mockResolvedValue({ id: 123, name: "Test User", @@ -367,8 +370,11 @@ describe("full reset compatibility with user 5h marker", () => { const { resetUserAllStatistics } = await import("@/actions/users"); const result = await resetUserAllStatistics(123); - expect(result.ok).toBe(false); - expect(result.errorCode).toBe(ERROR_CODES.CONNECTION_FAILED); + expect(result.ok).toBe(true); + expect(getRedisClientMock).not.toHaveBeenCalled(); + expect(enqueueUserStatisticsResetMock).toHaveBeenCalledWith(123, { + fixed5hKeyIds: [11], + }); expect(txUpdateSetMock).not.toHaveBeenCalled(); }); }); From f70d5635d02a5af8611412192728b2c92e50dad9 Mon Sep 17 00:00:00 2001 From: ding113 Date: Mon, 3 Aug 2026 05:01:10 +0800 Subject: [PATCH 17/18] fix(replay): preserve live attachments on durable reuse --- src/app/v1/_lib/proxy/replay/replay-spool.ts | 23 ++++++++++---------- tests/unit/proxy/replay-spool.test.ts | 10 ++++++--- tests/unit/proxy/replay-store.test.ts | 5 ++++- 3 files changed, 22 insertions(+), 16 deletions(-) diff --git a/src/app/v1/_lib/proxy/replay/replay-spool.ts b/src/app/v1/_lib/proxy/replay/replay-spool.ts index c14ccdbae..3691494c3 100644 --- a/src/app/v1/_lib/proxy/replay/replay-spool.ts +++ b/src/app/v1/_lib/proxy/replay/replay-spool.ts @@ -270,13 +270,6 @@ export class ReplaySpool { byteSize: this.totalBytes, sourceMessageRequestId: messageRequestId, }); - if (persistResult === "existing") { - await this.store.discardOwned(this.identity.replayId, this.ownerToken); - logger.info("[ReplaySpool] reused existing durable replay winner", { - replayId: this.identity.replayId.slice(0, 12), - }); - return; - } pgPersisted = true; const completed = await this.store.completeOwned( this.identity.replayId, @@ -286,11 +279,17 @@ export class ReplaySpool { if (!completed) { throw new Error("replay owner lease lost before completed meta"); } - logger.info("[ReplaySpool] replay entry completed", { - replayId: this.identity.replayId.slice(0, 12), - chunkCount: this.chunkCount, - byteSize: this.totalBytes, - }); + if (persistResult === "existing") { + logger.info("[ReplaySpool] reused existing durable replay winner", { + replayId: this.identity.replayId.slice(0, 12), + }); + } else { + logger.info("[ReplaySpool] replay entry completed", { + replayId: this.identity.replayId.slice(0, 12), + chunkCount: this.chunkCount, + byteSize: this.totalBytes, + }); + } } catch (error) { if (error instanceof ReplayDurableConflictError) { logger.warn("[ReplaySpool] discarded conflicting durable replay candidate", { diff --git a/tests/unit/proxy/replay-spool.test.ts b/tests/unit/proxy/replay-spool.test.ts index 3e502420d..f3988350e 100644 --- a/tests/unit/proxy/replay-spool.test.ts +++ b/tests/unit/proxy/replay-spool.test.ts @@ -440,15 +440,19 @@ describe("ReplaySpool：completeAfterBilling 终态屏障", () => { expect(getActiveReplaySpoolCount()).toBe(0); }); - it("复用已有 durable winner 时丢弃当前热层候选，不写 aborted", async () => { + it("复用已有 durable winner 时发布 completed 终态并保留 live attachment", async () => { storeControl.store.persistCompleted.mockResolvedValueOnce("existing"); const spool = makeSpool(); spool.observe(encoder.encode("data: a\n\n")); await spool.completeAfterBilling(7); - expect(storeControl.store.discardOwned).toHaveBeenCalledWith(identity.replayId, "owner-token"); - expect(storeControl.store.completeOwned).not.toHaveBeenCalled(); + expect(storeControl.store.completeOwned).toHaveBeenCalledWith( + identity.replayId, + "owner-token", + expect.objectContaining({ status: "completed", messageRequestId: 7, chunkCount: 1 }) + ); + expect(storeControl.store.discardOwned).not.toHaveBeenCalled(); expect(storeControl.store.abortOwned).not.toHaveBeenCalled(); }); diff --git a/tests/unit/proxy/replay-store.test.ts b/tests/unit/proxy/replay-store.test.ts index aa9d4c6ed..0b2b9dcc3 100644 --- a/tests/unit/proxy/replay-store.test.ts +++ b/tests/unit/proxy/replay-store.test.ts @@ -542,17 +542,20 @@ describe("ReplayStore：owner 租约", () => { expect(currentRedis().kv.has("cch:replay:owner:r1")).toBe(false); }); - it("completeOwned 仅在 token 匹配时原子写 completed meta 并释放租约", async () => { + it("completeOwned 仅在 token 匹配时原子写 completed meta、保留 chunks 并释放租约", async () => { const store = new ReplayStore(); const completedMeta = makeMeta({ status: "completed", chunkCount: 2 }); await store.tryClaimOwner("r1", "tok-a"); + await store.appendChunks("r1", ["first", "second"]); await expect(store.completeOwned("r1", "tok-other", completedMeta)).resolves.toBe(false); expect(currentRedis().kv.get("cch:replay:owner:r1")).toBe("tok-a"); await expect(store.getMeta("r1")).resolves.toBeNull(); + await expect(store.readChunks("r1", 0)).resolves.toEqual(["first", "second"]); await expect(store.completeOwned("r1", "tok-a", completedMeta)).resolves.toBe(true); await expect(store.getMeta("r1")).resolves.toEqual(completedMeta); + await expect(store.readChunks("r1", 0)).resolves.toEqual(["first", "second"]); expect(currentRedis().kv.has("cch:replay:owner:r1")).toBe(false); }); }); From 18c6e21a0763e2364078ab40ed53690873afc4b7 Mon Sep 17 00:00:00 2001 From: ding113 Date: Mon, 3 Aug 2026 05:34:03 +0800 Subject: [PATCH 18/18] fix(message): prevent reserved session identities from aliasing physical ledger rows Reserved canonical identities (prefix-affinity or owner-scoped session IDs) could match unrelated physical ledger rows via the session_id fallback in the OR branch of the lookup condition. For reserved identities the lookup now requires both the COALESCE expression and the explicit session_identity column to match, preserving the COALESCE expression index while avoiding aliasing. The canonical lookup used by aggregateSessionStats and aggregateMultipleSessionStats now also applies the stricter reserved-identity condition so owner-scoped queries do not silently pick up physical sessions that happen to share the same ID value. --- src/repository/message.ts | 20 ++++++---- ...e-aggregate-multiple-session-stats.test.ts | 40 +++++++++++++++++++ .../message-aggregate-session-stats.test.ts | 31 +++++++++++++- 3 files changed, 82 insertions(+), 9 deletions(-) diff --git a/src/repository/message.ts b/src/repository/message.ts index 48b5779a1..605614a31 100644 --- a/src/repository/message.ts +++ b/src/repository/message.ts @@ -37,14 +37,18 @@ const POST_TERMINAL_ROUTING_TRACE_ACK_TIMEOUT_MS = 3_000; const ledgerSessionIdentity = sql`COALESCE(${usageLedger.sessionIdentity}, ${usageLedger.sessionId})`; const messageSessionIdentity = sql`COALESCE(${messageRequest.sessionIdentity}, ${messageRequest.sessionId})`; +function ledgerCanonicalSessionCondition(identity: string) { + return isReservedSessionIdentity(identity) + ? and(eq(ledgerSessionIdentity, identity), eq(usageLedger.sessionIdentity, identity)) + : eq(ledgerSessionIdentity, identity); +} + function ledgerSessionLookupForOwner(identityOrPhysicalId: string, ownerUserId?: number) { - const canonicalCondition = isReservedSessionIdentity(identityOrPhysicalId) - ? eq(usageLedger.sessionIdentity, identityOrPhysicalId) - : eq(ledgerSessionIdentity, identityOrPhysicalId); - const lookupCondition = - ownerUserId !== undefined || !isReservedSessionIdentity(identityOrPhysicalId) - ? or(canonicalCondition, eq(usageLedger.sessionId, identityOrPhysicalId)) - : canonicalCondition; + const reservedIdentity = isReservedSessionIdentity(identityOrPhysicalId); + const canonicalCondition = ledgerCanonicalSessionCondition(identityOrPhysicalId); + const lookupCondition = reservedIdentity + ? canonicalCondition + : or(canonicalCondition, eq(usageLedger.sessionId, identityOrPhysicalId)); return and( lookupCondition, @@ -53,7 +57,7 @@ function ledgerSessionLookupForOwner(identityOrPhysicalId: string, ownerUserId?: } function ledgerCanonicalSessionLookup(identity: string, ownerUserId: number) { - return and(eq(ledgerSessionIdentity, identity), eq(usageLedger.userId, ownerUserId)); + return and(ledgerCanonicalSessionCondition(identity), eq(usageLedger.userId, ownerUserId)); } function messageSessionLookup(identityOrPhysicalId: string, ownerUserId?: number) { diff --git a/tests/unit/repository/message-aggregate-multiple-session-stats.test.ts b/tests/unit/repository/message-aggregate-multiple-session-stats.test.ts index 3db308eb4..b77ac0bca 100644 --- a/tests/unit/repository/message-aggregate-multiple-session-stats.test.ts +++ b/tests/unit/repository/message-aggregate-multiple-session-stats.test.ts @@ -385,4 +385,44 @@ describe("message repository aggregateMultipleSessionStats", () => { expect(ownerQuery).not.toContain("session_identity = sid or session_id = sid"); expect(ownerQuery).toContain("created_at desc nulls last, id desc"); }); + + test.each(["pfx:scope:root", "sid:owner-root"])( + "aggregates reserved canonical identity from explicit ledger identity only: %s", + async (canonicalId) => { + const stats = createDrizzleQuery([statsRow(canonicalId, 1)]); + const providerList = createDrizzleQuery([]); + const modelList = createDrizzleQuery([]); + const cacheTtlList = createDrizzleQuery([]); + boundary.select.mockReturnValueOnce(stats); + boundary.selectDistinct + .mockReturnValueOnce(providerList) + .mockReturnValueOnce(modelList) + .mockReturnValueOnce(cacheTtlList); + boundary.execute.mockResolvedValueOnce([ + { + requested_session_id: canonicalId, + session_id: canonicalId, + session_identity_kind: "prefix_affinity", + session_fingerprint: "root", + user_name: "Alice", + user_id: 1, + key_name: "Key A", + key_id: 101, + user_agent: null, + api_type: "claude", + }, + ]); + + await aggregateMultipleSessionStats([canonicalId]); + + for (const query of [stats, providerList, modelList, cacheTtlList]) { + const whereSql = sqlText(query.trace.where).toLowerCase(); + expect(whereSql).toContain("coalesce"); + expect(whereSql).toContain("session_identity"); + expect(whereSql.match(new RegExp(canonicalId, "g"))).toHaveLength(2); + expect(whereSql).not.toContain(`session_id = ${canonicalId}`); + expect(whereSql).toContain("user_id"); + } + } + ); }); diff --git a/tests/unit/repository/message-aggregate-session-stats.test.ts b/tests/unit/repository/message-aggregate-session-stats.test.ts index 4f955dcf5..e8e17ce05 100644 --- a/tests/unit/repository/message-aggregate-session-stats.test.ts +++ b/tests/unit/repository/message-aggregate-session-stats.test.ts @@ -137,11 +137,40 @@ describe("message repository aggregateSessionStats", () => { await aggregateSessionStats("pfx:scope123:fp-deep"); const whereSql = sqlText(stats.trace.where); + expect(whereSql).toContain("coalesce"); expect(whereSql).toContain("session_identity"); expect(whereSql).toContain("session_id"); - expect(whereSql.match(/pfx:scope123:fp-deep/g)).toHaveLength(1); + expect(whereSql.match(/pfx:scope123:fp-deep/g)).toHaveLength(2); + expect(whereSql).not.toContain("session_id = pfx:scope123:fp-deep"); }); + test.each(["pfx:scope123:fp-deep", "sid:owner-session"])( + "owner-scoped reserved identity does not alias a physical ledger Session: %s", + async (reservedIdentity) => { + const stats = createDrizzleQuery([]); + const providerList = createDrizzleQuery([]); + const modelList = createDrizzleQuery([]); + const cacheTtlList = createDrizzleQuery([]); + boundary.select.mockReturnValueOnce(stats).mockReturnValueOnce(createDrizzleQuery([])); + boundary.selectDistinct + .mockReturnValueOnce(providerList) + .mockReturnValueOnce(modelList) + .mockReturnValueOnce(cacheTtlList); + + await aggregateSessionStats(reservedIdentity, 17); + + for (const query of [stats, providerList, modelList, cacheTtlList]) { + const whereSql = sqlText(query.trace.where); + expect(whereSql).toContain("coalesce"); + expect(whereSql).toContain("session_identity"); + expect(whereSql.match(new RegExp(reservedIdentity, "g"))).toHaveLength(2); + expect(whereSql).not.toContain(`session_id = ${reservedIdentity}`); + expect(whereSql).toContain("user_id"); + expect(whereSql).toContain("17"); + } + } + ); + test("returns populated statistics and preserves a single cache TTL", async () => { const queries = queuePopulatedAggregate(["1h"]);