From b37db1503957a3ff380dd11ecdaa8a52c5e3bdbd Mon Sep 17 00:00:00 2001 From: ding113 Date: Fri, 7 Aug 2026 03:16:39 +0800 Subject: [PATCH 01/12] feat(replay): add configurable cache TTL system setting Introduce replayCacheTtlMinutes (5-120 min, default 30) as a database-backed system setting controlling how long completed Replay payloads remain reusable in the PostgreSQL durable layer. The Redis hot-layer TTL is capped to the same window so both tiers expire in sync. This replaces the removed REPLAY_COMPLETED_TTL_SECONDS environment variable, moving the durable TTL into the admin-editable settings surface with full validation, i18n labels, API schema, and UI input. Also fix the replay cleanup query to bind the cutoff Date through sql.param for correct PostgreSQL type coercion, and preserve wrapped database error causes in the cleanup scheduler logging. --- drizzle/0119_tiresome_banshee.sql | 1 + drizzle/meta/0119_snapshot.json | 5404 +++++++++++++++++ drizzle/meta/_journal.json | 7 + messages/en/settings/config.json | 4 + messages/ja/settings/config.json | 4 + messages/ru/settings/config.json | 4 + messages/zh-CN/settings/config.json | 4 + messages/zh-TW/settings/config.json | 4 + src/actions/system-config.ts | 20 +- .../_components/system-settings-form.tsx | 73 +- src/app/[locale]/settings/config/page.tsx | 1 + src/app/api/v1/resources/system/router.ts | 4 +- src/app/v1/_lib/proxy/replay/replay-store.ts | 27 +- src/drizzle/schema.ts | 2 + src/instrumentation.ts | 16 +- src/lib/api-client/v1/openapi-types.gen.ts | 6 + src/lib/api/v1/schemas/system-config.ts | 10 + src/lib/config/env.schema.ts | 2 - src/lib/config/system-settings-cache.ts | 4 + src/lib/system-settings/proxy-runtime.ts | 5 + src/lib/validation/replay-settings.ts | 16 + src/lib/validation/schemas.ts | 12 + src/repository/_shared/transformers.test.ts | 5 + src/repository/_shared/transformers.ts | 2 + src/repository/system-config.ts | 14 + src/types/system-config.ts | 4 + tests/api/v1/system/system-config.test.ts | 21 +- tests/unit/actions/system-config-save.test.ts | 20 + .../instrumentation-replay-cleanup.test.ts | 48 +- tests/unit/proxy/replay-store.test.ts | 44 +- .../proxy/stream-gate-mode-resolution.test.ts | 44 +- .../system-config-degradation-ladder.test.ts | 27 +- ...stem-config-update-missing-columns.test.ts | 27 +- ...ettings-form-replay-cache-toggles.test.tsx | 46 + 34 files changed, 5856 insertions(+), 76 deletions(-) create mode 100644 drizzle/0119_tiresome_banshee.sql create mode 100644 drizzle/meta/0119_snapshot.json create mode 100644 src/lib/validation/replay-settings.ts diff --git a/drizzle/0119_tiresome_banshee.sql b/drizzle/0119_tiresome_banshee.sql new file mode 100644 index 000000000..4b38b09ee --- /dev/null +++ b/drizzle/0119_tiresome_banshee.sql @@ -0,0 +1 @@ +ALTER TABLE "system_settings" ADD COLUMN "replay_cache_ttl_minutes" integer DEFAULT 30 NOT NULL; \ No newline at end of file diff --git a/drizzle/meta/0119_snapshot.json b/drizzle/meta/0119_snapshot.json new file mode 100644 index 000000000..39af8c9a9 --- /dev/null +++ b/drizzle/meta/0119_snapshot.json @@ -0,0 +1,5404 @@ +{ + "id": "12d909a0-d12b-4617-b7af-5a64a50201a7", + "prevId": "0a61db74-24d2-408d-b247-974f2413fa50", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.audit_log": { + "name": "audit_log", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "action_category": { + "name": "action_category", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true + }, + "action_type": { + "name": "action_type", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true + }, + "target_type": { + "name": "target_type", + "type": "varchar(32)", + "primaryKey": false, + "notNull": false + }, + "target_id": { + "name": "target_id", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "target_name": { + "name": "target_name", + "type": "varchar(256)", + "primaryKey": false, + "notNull": false + }, + "before_value": { + "name": "before_value", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "after_value": { + "name": "after_value", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "operator_user_id": { + "name": "operator_user_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "operator_user_name": { + "name": "operator_user_name", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false + }, + "operator_key_id": { + "name": "operator_key_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "operator_key_name": { + "name": "operator_key_name", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false + }, + "operator_ip": { + "name": "operator_ip", + "type": "varchar(45)", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false + }, + "success": { + "name": "success", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_audit_log_category_created_at": { + "name": "idx_audit_log_category_created_at", + "columns": [ + { + "expression": "action_category", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_audit_log_operator_user_created_at": { + "name": "idx_audit_log_operator_user_created_at", + "columns": [ + { + "expression": "operator_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"audit_log\".\"operator_user_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_audit_log_operator_ip_created_at": { + "name": "idx_audit_log_operator_ip_created_at", + "columns": [ + { + "expression": "operator_ip", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"audit_log\".\"operator_ip\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_audit_log_target": { + "name": "idx_audit_log_target", + "columns": [ + { + "expression": "target_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"audit_log\".\"target_type\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_audit_log_created_at_id": { + "name": "idx_audit_log_created_at_id", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.cloud_pricing_catalog": { + "name": "cloud_pricing_catalog", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "version": { + "name": "version", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true + }, + "currency": { + "name": "currency", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "default": "'USD'" + }, + "refreshed_at": { + "name": "refreshed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "providers": { + "name": "providers", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "vendors": { + "name": "vendors", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "model_count": { + "name": "model_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "synced_at": { + "name": "synced_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.error_rules": { + "name": "error_rules", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "pattern": { + "name": "pattern", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "match_type": { + "name": "match_type", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'regex'" + }, + "category": { + "name": "category", + "type": "varchar(50)", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "override_response": { + "name": "override_response", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "override_status_code": { + "name": "override_status_code", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "is_enabled": { + "name": "is_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "is_default": { + "name": "is_default", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "priority": { + "name": "priority", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "idx_error_rules_enabled": { + "name": "idx_error_rules_enabled", + "columns": [ + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "priority", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "unique_pattern": { + "name": "unique_pattern", + "columns": [ + { + "expression": "pattern", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_category": { + "name": "idx_category", + "columns": [ + { + "expression": "category", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_match_type": { + "name": "idx_match_type", + "columns": [ + { + "expression": "match_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.keys": { + "name": "keys", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "is_enabled": { + "name": "is_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "can_login_web_ui": { + "name": "can_login_web_ui", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "limit_5h_usd": { + "name": "limit_5h_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "limit_5h_reset_mode": { + "name": "limit_5h_reset_mode", + "type": "daily_reset_mode", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'rolling'" + }, + "limit_daily_usd": { + "name": "limit_daily_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "daily_reset_mode": { + "name": "daily_reset_mode", + "type": "daily_reset_mode", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'fixed'" + }, + "daily_reset_time": { + "name": "daily_reset_time", + "type": "varchar(5)", + "primaryKey": false, + "notNull": true, + "default": "'00:00'" + }, + "limit_weekly_usd": { + "name": "limit_weekly_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "limit_monthly_usd": { + "name": "limit_monthly_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "limit_total_usd": { + "name": "limit_total_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "cost_reset_at": { + "name": "cost_reset_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "limit_concurrent_sessions": { + "name": "limit_concurrent_sessions", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "provider_group": { + "name": "provider_group", + "type": "varchar(200)", + "primaryKey": false, + "notNull": false, + "default": "'default'" + }, + "cache_ttl_preference": { + "name": "cache_ttl_preference", + "type": "varchar(10)", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "idx_keys_user_id": { + "name": "idx_keys_user_id", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_keys_key": { + "name": "idx_keys_key", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_keys_created_at": { + "name": "idx_keys_created_at", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_keys_deleted_at": { + "name": "idx_keys_deleted_at", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.message_request": { + "name": "message_request", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "model": { + "name": "model", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "cost_usd": { + "name": "cost_usd", + "type": "numeric(21, 15)", + "primaryKey": false, + "notNull": false, + "default": "'0'" + }, + "cost_multiplier": { + "name": "cost_multiplier", + "type": "numeric(10, 4)", + "primaryKey": false, + "notNull": false + }, + "group_cost_multiplier": { + "name": "group_cost_multiplier", + "type": "numeric(10, 4)", + "primaryKey": false, + "notNull": false + }, + "cost_breakdown": { + "name": "cost_breakdown", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "session_id": { + "name": "session_id", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "session_identity": { + "name": "session_identity", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "session_identity_kind": { + "name": "session_identity_kind", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "affinity_scope_tag": { + "name": "affinity_scope_tag", + "type": "varchar(16)", + "primaryKey": false, + "notNull": false + }, + "affinity_fingerprint": { + "name": "affinity_fingerprint", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "affinity_fingerprint_chain": { + "name": "affinity_fingerprint_chain", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "is_replay": { + "name": "is_replay", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "replay_source_request_id": { + "name": "replay_source_request_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "request_sequence": { + "name": "request_sequence", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 1 + }, + "provider_chain": { + "name": "provider_chain", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "routing_trace": { + "name": "routing_trace", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "status_code": { + "name": "status_code", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "api_type": { + "name": "api_type", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "endpoint": { + "name": "endpoint", + "type": "varchar(256)", + "primaryKey": false, + "notNull": false + }, + "original_model": { + "name": "original_model", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false + }, + "actual_response_model": { + "name": "actual_response_model", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false + }, + "input_tokens": { + "name": "input_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "output_tokens": { + "name": "output_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "ttfb_ms": { + "name": "ttfb_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "first_byte_ms": { + "name": "first_byte_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "cache_creation_input_tokens": { + "name": "cache_creation_input_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "cache_read_input_tokens": { + "name": "cache_read_input_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "cache_creation_5m_input_tokens": { + "name": "cache_creation_5m_input_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "cache_creation_1h_input_tokens": { + "name": "cache_creation_1h_input_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "cache_ttl_applied": { + "name": "cache_ttl_applied", + "type": "varchar(10)", + "primaryKey": false, + "notNull": false + }, + "context_1m_applied": { + "name": "context_1m_applied", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "swap_cache_ttl_applied": { + "name": "swap_cache_ttl_applied", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "special_settings": { + "name": "special_settings", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "hedge_losers": { + "name": "hedge_losers", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error_stack": { + "name": "error_stack", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error_cause": { + "name": "error_cause", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "blocked_by": { + "name": "blocked_by", + "type": "varchar(50)", + "primaryKey": false, + "notNull": false + }, + "blocked_reason": { + "name": "blocked_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false + }, + "client_ip": { + "name": "client_ip", + "type": "varchar(45)", + "primaryKey": false, + "notNull": false + }, + "messages_count": { + "name": "messages_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "cache_compatibility_key": { + "name": "cache_compatibility_key", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "cache_score_eligible": { + "name": "cache_score_eligible", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "cache_score_excluded_reason": { + "name": "cache_score_excluded_reason", + "type": "varchar(32)", + "primaryKey": false, + "notNull": false + }, + "theoretical_cache_tokens": { + "name": "theoretical_cache_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "cache_ttl_bucket": { + "name": "cache_ttl_bucket", + "type": "varchar(10)", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "idx_message_request_user_date_cost": { + "name": "idx_message_request_user_date_cost", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "cost_usd", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_user_created_at_cost_stats": { + "name": "idx_message_request_user_created_at_cost_stats", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "cost_usd", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND (\"message_request\".\"blocked_by\" IS NULL OR \"message_request\".\"blocked_by\" <> 'warmup')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_user_query": { + "name": "idx_message_request_user_query", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_provider_created_at_active": { + "name": "idx_message_request_provider_created_at_active", + "columns": [ + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND (\"message_request\".\"blocked_by\" IS NULL OR \"message_request\".\"blocked_by\" <> 'warmup')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_provider_created_at_finalized_active": { + "name": "idx_message_request_provider_created_at_finalized_active", + "columns": [ + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND \"message_request\".\"status_code\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_proxy_status_active": { + "name": "idx_message_request_proxy_status_active", + "columns": [ + { + "expression": "\"created_at\" DESC NULLS LAST", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND \"message_request\".\"is_replay\" = false AND \"message_request\".\"status_code\" IS NULL AND (\"message_request\".\"blocked_by\" IS NULL OR \"message_request\".\"blocked_by\" <> 'warmup')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_proxy_status_latest": { + "name": "idx_message_request_proxy_status_latest", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "\"updated_at\" DESC NULLS LAST", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND \"message_request\".\"is_replay\" = false AND \"message_request\".\"status_code\" IS NOT NULL AND (\"message_request\".\"blocked_by\" IS NULL OR \"message_request\".\"blocked_by\" <> 'warmup')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_session_id": { + "name": "idx_message_request_session_id", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_session_id_prefix": { + "name": "idx_message_request_session_id_prefix", + "columns": [ + { + "expression": "\"session_id\" varchar_pattern_ops", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND (\"message_request\".\"blocked_by\" IS NULL OR \"message_request\".\"blocked_by\" <> 'warmup')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_session_seq": { + "name": "idx_message_request_session_seq", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "request_sequence", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_session_identity_created_at": { + "name": "idx_message_request_session_identity_created_at", + "columns": [ + { + "expression": "COALESCE(\"session_identity\", \"session_id\")", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "\"created_at\" DESC NULLS LAST", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_endpoint": { + "name": "idx_message_request_endpoint", + "columns": [ + { + "expression": "endpoint", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_blocked_by": { + "name": "idx_message_request_blocked_by", + "columns": [ + { + "expression": "blocked_by", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_provider_id": { + "name": "idx_message_request_provider_id", + "columns": [ + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_user_id": { + "name": "idx_message_request_user_id", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_key": { + "name": "idx_message_request_key", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_key_created_at_id": { + "name": "idx_message_request_key_created_at_id", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_key_model_active": { + "name": "idx_message_request_key_model_active", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "model", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND \"message_request\".\"model\" IS NOT NULL AND (\"message_request\".\"blocked_by\" IS NULL OR \"message_request\".\"blocked_by\" <> 'warmup')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_key_endpoint_active": { + "name": "idx_message_request_key_endpoint_active", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "endpoint", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND \"message_request\".\"endpoint\" IS NOT NULL AND (\"message_request\".\"blocked_by\" IS NULL OR \"message_request\".\"blocked_by\" <> 'warmup')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_created_at_id_active": { + "name": "idx_message_request_created_at_id_active", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_model_active": { + "name": "idx_message_request_model_active", + "columns": [ + { + "expression": "model", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND \"message_request\".\"model\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_status_code_active": { + "name": "idx_message_request_status_code_active", + "columns": [ + { + "expression": "status_code", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND \"message_request\".\"status_code\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_created_at": { + "name": "idx_message_request_created_at", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_deleted_at": { + "name": "idx_message_request_deleted_at", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_key_last_active": { + "name": "idx_message_request_key_last_active", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND (\"message_request\".\"blocked_by\" IS NULL OR \"message_request\".\"blocked_by\" <> 'warmup')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_key_cost_active": { + "name": "idx_message_request_key_cost_active", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "cost_usd", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND (\"message_request\".\"blocked_by\" IS NULL OR \"message_request\".\"blocked_by\" <> 'warmup')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_session_user_info": { + "name": "idx_message_request_session_user_info", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_client_ip_created_at": { + "name": "idx_message_request_client_ip_created_at", + "columns": [ + { + "expression": "client_ip", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND \"message_request\".\"client_ip\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.model_prices": { + "name": "model_prices", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "model_name": { + "name": "model_name", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "price_data": { + "name": "price_data", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'cloud'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "idx_model_prices_latest": { + "name": "idx_model_prices_latest", + "columns": [ + { + "expression": "model_name", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_model_prices_model_name": { + "name": "idx_model_prices_model_name", + "columns": [ + { + "expression": "model_name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_model_prices_created_at": { + "name": "idx_model_prices_created_at", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_model_prices_source": { + "name": "idx_model_prices_source", + "columns": [ + { + "expression": "source", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_model_prices_vendor": { + "name": "idx_model_prices_vendor", + "columns": [ + { + "expression": "((\"price_data\" ->> 'vendor'))", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_model_prices_aliases": { + "name": "idx_model_prices_aliases", + "columns": [ + { + "expression": "((\"price_data\" -> 'aliases'))", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.notification_settings": { + "name": "notification_settings", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "use_legacy_mode": { + "name": "use_legacy_mode", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "circuit_breaker_enabled": { + "name": "circuit_breaker_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "circuit_breaker_webhook": { + "name": "circuit_breaker_webhook", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false + }, + "daily_leaderboard_enabled": { + "name": "daily_leaderboard_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "daily_leaderboard_webhook": { + "name": "daily_leaderboard_webhook", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false + }, + "daily_leaderboard_time": { + "name": "daily_leaderboard_time", + "type": "varchar(10)", + "primaryKey": false, + "notNull": false, + "default": "'09:00'" + }, + "daily_leaderboard_top_n": { + "name": "daily_leaderboard_top_n", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 5 + }, + "cost_alert_enabled": { + "name": "cost_alert_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "cost_alert_webhook": { + "name": "cost_alert_webhook", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false + }, + "cost_alert_threshold": { + "name": "cost_alert_threshold", + "type": "numeric(5, 2)", + "primaryKey": false, + "notNull": false, + "default": "'0.80'" + }, + "cost_alert_check_interval": { + "name": "cost_alert_check_interval", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 60 + }, + "cache_hit_rate_alert_enabled": { + "name": "cache_hit_rate_alert_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "cache_hit_rate_alert_webhook": { + "name": "cache_hit_rate_alert_webhook", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false + }, + "cache_hit_rate_alert_window_mode": { + "name": "cache_hit_rate_alert_window_mode", + "type": "varchar(10)", + "primaryKey": false, + "notNull": false, + "default": "'auto'" + }, + "cache_hit_rate_alert_check_interval": { + "name": "cache_hit_rate_alert_check_interval", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 5 + }, + "cache_hit_rate_alert_historical_lookback_days": { + "name": "cache_hit_rate_alert_historical_lookback_days", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 7 + }, + "cache_hit_rate_alert_min_eligible_requests": { + "name": "cache_hit_rate_alert_min_eligible_requests", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 20 + }, + "cache_hit_rate_alert_min_eligible_tokens": { + "name": "cache_hit_rate_alert_min_eligible_tokens", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "cache_hit_rate_alert_abs_min": { + "name": "cache_hit_rate_alert_abs_min", + "type": "numeric(5, 4)", + "primaryKey": false, + "notNull": false, + "default": "'0.05'" + }, + "cache_hit_rate_alert_drop_rel": { + "name": "cache_hit_rate_alert_drop_rel", + "type": "numeric(5, 4)", + "primaryKey": false, + "notNull": false, + "default": "'0.3'" + }, + "cache_hit_rate_alert_drop_abs": { + "name": "cache_hit_rate_alert_drop_abs", + "type": "numeric(5, 4)", + "primaryKey": false, + "notNull": false, + "default": "'0.1'" + }, + "cache_hit_rate_alert_cooldown_minutes": { + "name": "cache_hit_rate_alert_cooldown_minutes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 30 + }, + "cache_hit_rate_alert_top_n": { + "name": "cache_hit_rate_alert_top_n", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 10 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.notification_target_bindings": { + "name": "notification_target_bindings", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "notification_type": { + "name": "notification_type", + "type": "notification_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "target_id": { + "name": "target_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "is_enabled": { + "name": "is_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "schedule_cron": { + "name": "schedule_cron", + "type": "varchar(100)", + "primaryKey": false, + "notNull": false + }, + "schedule_timezone": { + "name": "schedule_timezone", + "type": "varchar(50)", + "primaryKey": false, + "notNull": false + }, + "template_override": { + "name": "template_override", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "unique_notification_target_binding": { + "name": "unique_notification_target_binding", + "columns": [ + { + "expression": "notification_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_notification_bindings_type": { + "name": "idx_notification_bindings_type", + "columns": [ + { + "expression": "notification_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_notification_bindings_target": { + "name": "idx_notification_bindings_target", + "columns": [ + { + "expression": "target_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "notification_target_bindings_target_id_webhook_targets_id_fk": { + "name": "notification_target_bindings_target_id_webhook_targets_id_fk", + "tableFrom": "notification_target_bindings", + "tableTo": "webhook_targets", + "columnsFrom": [ + "target_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.provider_batch_apply_operations": { + "name": "provider_batch_apply_operations", + "schema": "", + "columns": { + "claim_key": { + "name": "claim_key", + "type": "varchar(256)", + "primaryKey": true, + "notNull": true + }, + "preview_token": { + "name": "preview_token", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "payload_fingerprint": { + "name": "payload_fingerprint", + "type": "varchar(128)", + "primaryKey": false, + "notNull": true + }, + "operation_id": { + "name": "operation_id", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "undo_token": { + "name": "undo_token", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "undo_expires_at": { + "name": "undo_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "undo_consumed_at": { + "name": "undo_consumed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true + }, + "result": { + "name": "result", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "uniq_provider_batch_apply_operations_preview_token": { + "name": "uniq_provider_batch_apply_operations_preview_token", + "columns": [ + { + "expression": "preview_token", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "uniq_provider_batch_apply_operations_operation_id": { + "name": "uniq_provider_batch_apply_operations_operation_id", + "columns": [ + { + "expression": "operation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "uniq_provider_batch_apply_operations_undo_token": { + "name": "uniq_provider_batch_apply_operations_undo_token", + "columns": [ + { + "expression": "undo_token", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_provider_batch_apply_operations_expires_at": { + "name": "idx_provider_batch_apply_operations_expires_at", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.provider_cache_effectiveness": { + "name": "provider_cache_effectiveness", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "model": { + "name": "model", + "type": "varchar(128)", + "primaryKey": false, + "notNull": true + }, + "cache_ttl_bucket": { + "name": "cache_ttl_bucket", + "type": "varchar(10)", + "primaryKey": false, + "notNull": true + }, + "window_start": { + "name": "window_start", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "window_end": { + "name": "window_end", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "sample_count": { + "name": "sample_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "eligible_count": { + "name": "eligible_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "theoretical_cache_tokens": { + "name": "theoretical_cache_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "observed_cache_read_tokens": { + "name": "observed_cache_read_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "raw_effectiveness_bp": { + "name": "raw_effectiveness_bp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "confidence_bp": { + "name": "confidence_bp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "effectiveness_bp": { + "name": "effectiveness_bp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "idx_provider_cache_effectiveness_window": { + "name": "idx_provider_cache_effectiveness_window", + "columns": [ + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "model", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "window_start", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.provider_endpoint_probe_logs": { + "name": "provider_endpoint_probe_logs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "endpoint_id": { + "name": "endpoint_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'scheduled'" + }, + "ok": { + "name": "ok", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "status_code": { + "name": "status_code", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "latency_ms": { + "name": "latency_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "error_type": { + "name": "error_type", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "idx_provider_endpoint_probe_logs_endpoint_created_at": { + "name": "idx_provider_endpoint_probe_logs_endpoint_created_at", + "columns": [ + { + "expression": "endpoint_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_provider_endpoint_probe_logs_created_at": { + "name": "idx_provider_endpoint_probe_logs_created_at", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "provider_endpoint_probe_logs_endpoint_id_provider_endpoints_id_fk": { + "name": "provider_endpoint_probe_logs_endpoint_id_provider_endpoints_id_fk", + "tableFrom": "provider_endpoint_probe_logs", + "tableTo": "provider_endpoints", + "columnsFrom": [ + "endpoint_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.provider_endpoints": { + "name": "provider_endpoints", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "vendor_id": { + "name": "vendor_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "provider_type": { + "name": "provider_type", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'claude'" + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "label": { + "name": "label", + "type": "varchar(200)", + "primaryKey": false, + "notNull": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "is_enabled": { + "name": "is_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "last_probed_at": { + "name": "last_probed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_probe_ok": { + "name": "last_probe_ok", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "last_probe_status_code": { + "name": "last_probe_status_code", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "last_probe_latency_ms": { + "name": "last_probe_latency_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "last_probe_error_type": { + "name": "last_probe_error_type", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "last_probe_error_message": { + "name": "last_probe_error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "uniq_provider_endpoints_vendor_type_url": { + "name": "uniq_provider_endpoints_vendor_type_url", + "columns": [ + { + "expression": "vendor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "url", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"provider_endpoints\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_provider_endpoints_vendor_type": { + "name": "idx_provider_endpoints_vendor_type", + "columns": [ + { + "expression": "vendor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"provider_endpoints\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_provider_endpoints_enabled": { + "name": "idx_provider_endpoints_enabled", + "columns": [ + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "vendor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"provider_endpoints\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_provider_endpoints_pick_enabled": { + "name": "idx_provider_endpoints_pick_enabled", + "columns": [ + { + "expression": "vendor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "sort_order", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"provider_endpoints\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_provider_endpoints_created_at": { + "name": "idx_provider_endpoints_created_at", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_provider_endpoints_deleted_at": { + "name": "idx_provider_endpoints_deleted_at", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "provider_endpoints_vendor_id_provider_vendors_id_fk": { + "name": "provider_endpoints_vendor_id_provider_vendors_id_fk", + "tableFrom": "provider_endpoints", + "tableTo": "provider_vendors", + "columnsFrom": [ + "vendor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.provider_groups": { + "name": "provider_groups", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(200)", + "primaryKey": false, + "notNull": true + }, + "cost_multiplier": { + "name": "cost_multiplier", + "type": "numeric(10, 4)", + "primaryKey": false, + "notNull": true, + "default": "'1.0'" + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "provider_groups_name_unique": { + "name": "provider_groups_name_unique", + "nullsNotDistinct": false, + "columns": [ + "name" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.provider_vendors": { + "name": "provider_vendors", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "website_domain": { + "name": "website_domain", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "varchar(200)", + "primaryKey": false, + "notNull": false + }, + "website_url": { + "name": "website_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "favicon_url": { + "name": "favicon_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "uniq_provider_vendors_website_domain": { + "name": "uniq_provider_vendors_website_domain", + "columns": [ + { + "expression": "website_domain", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_provider_vendors_created_at": { + "name": "idx_provider_vendors_created_at", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.providers": { + "name": "providers", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "url": { + "name": "url", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "provider_vendor_id": { + "name": "provider_vendor_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "is_enabled": { + "name": "is_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "weight": { + "name": "weight", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "priority": { + "name": "priority", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "group_priorities": { + "name": "group_priorities", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'null'::jsonb" + }, + "cost_multiplier": { + "name": "cost_multiplier", + "type": "numeric(10, 4)", + "primaryKey": false, + "notNull": false, + "default": "'1.0'" + }, + "group_tag": { + "name": "group_tag", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "provider_type": { + "name": "provider_type", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'claude'" + }, + "preserve_client_ip": { + "name": "preserve_client_ip", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "disable_session_reuse": { + "name": "disable_session_reuse", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "model_redirects": { + "name": "model_redirects", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "allowed_models": { + "name": "allowed_models", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'null'::jsonb" + }, + "allowed_clients": { + "name": "allowed_clients", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "blocked_clients": { + "name": "blocked_clients", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "active_time_start": { + "name": "active_time_start", + "type": "varchar(5)", + "primaryKey": false, + "notNull": false + }, + "active_time_end": { + "name": "active_time_end", + "type": "varchar(5)", + "primaryKey": false, + "notNull": false + }, + "codex_instructions_strategy": { + "name": "codex_instructions_strategy", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false, + "default": "'auto'" + }, + "mcp_passthrough_type": { + "name": "mcp_passthrough_type", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'none'" + }, + "mcp_passthrough_url": { + "name": "mcp_passthrough_url", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false + }, + "limit_5h_usd": { + "name": "limit_5h_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "limit_5h_reset_mode": { + "name": "limit_5h_reset_mode", + "type": "daily_reset_mode", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'rolling'" + }, + "limit_daily_usd": { + "name": "limit_daily_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "daily_reset_mode": { + "name": "daily_reset_mode", + "type": "daily_reset_mode", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'fixed'" + }, + "daily_reset_time": { + "name": "daily_reset_time", + "type": "varchar(5)", + "primaryKey": false, + "notNull": true, + "default": "'00:00'" + }, + "limit_weekly_usd": { + "name": "limit_weekly_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "limit_monthly_usd": { + "name": "limit_monthly_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "limit_total_usd": { + "name": "limit_total_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "total_cost_reset_at": { + "name": "total_cost_reset_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "limit_concurrent_sessions": { + "name": "limit_concurrent_sessions", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "max_retry_attempts": { + "name": "max_retry_attempts", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "circuit_breaker_failure_threshold": { + "name": "circuit_breaker_failure_threshold", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 5 + }, + "circuit_breaker_open_duration": { + "name": "circuit_breaker_open_duration", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 1800000 + }, + "circuit_breaker_half_open_success_threshold": { + "name": "circuit_breaker_half_open_success_threshold", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 2 + }, + "proxy_url": { + "name": "proxy_url", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false + }, + "proxy_fallback_to_direct": { + "name": "proxy_fallback_to_direct", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "custom_headers": { + "name": "custom_headers", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "first_byte_timeout_streaming_ms": { + "name": "first_byte_timeout_streaming_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "streaming_idle_timeout_ms": { + "name": "streaming_idle_timeout_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "request_timeout_non_streaming_ms": { + "name": "request_timeout_non_streaming_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "website_url": { + "name": "website_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "favicon_url": { + "name": "favicon_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cache_ttl_preference": { + "name": "cache_ttl_preference", + "type": "varchar(10)", + "primaryKey": false, + "notNull": false + }, + "swap_cache_ttl_billing": { + "name": "swap_cache_ttl_billing", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "context_1m_preference": { + "name": "context_1m_preference", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "codex_reasoning_effort_preference": { + "name": "codex_reasoning_effort_preference", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "codex_reasoning_summary_preference": { + "name": "codex_reasoning_summary_preference", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "codex_text_verbosity_preference": { + "name": "codex_text_verbosity_preference", + "type": "varchar(10)", + "primaryKey": false, + "notNull": false + }, + "codex_parallel_tool_calls_preference": { + "name": "codex_parallel_tool_calls_preference", + "type": "varchar(10)", + "primaryKey": false, + "notNull": false + }, + "codex_image_generation_preference": { + "name": "codex_image_generation_preference", + "type": "varchar(10)", + "primaryKey": false, + "notNull": false + }, + "codex_service_tier_preference": { + "name": "codex_service_tier_preference", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "anthropic_max_tokens_preference": { + "name": "anthropic_max_tokens_preference", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "anthropic_thinking_budget_preference": { + "name": "anthropic_thinking_budget_preference", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "anthropic_adaptive_thinking": { + "name": "anthropic_adaptive_thinking", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'null'::jsonb" + }, + "gemini_google_search_preference": { + "name": "gemini_google_search_preference", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "tpm": { + "name": "tpm", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "rpm": { + "name": "rpm", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "rpd": { + "name": "rpd", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "cc": { + "name": "cc", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "idx_providers_enabled_priority": { + "name": "idx_providers_enabled_priority", + "columns": [ + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "priority", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "weight", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"providers\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_providers_group": { + "name": "idx_providers_group", + "columns": [ + { + "expression": "group_tag", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"providers\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_providers_vendor_type_url_active": { + "name": "idx_providers_vendor_type_url_active", + "columns": [ + { + "expression": "provider_vendor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "url", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"providers\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_providers_created_at": { + "name": "idx_providers_created_at", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_providers_deleted_at": { + "name": "idx_providers_deleted_at", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_providers_vendor_type": { + "name": "idx_providers_vendor_type", + "columns": [ + { + "expression": "provider_vendor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"providers\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_providers_enabled_vendor_type": { + "name": "idx_providers_enabled_vendor_type", + "columns": [ + { + "expression": "provider_vendor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"providers\".\"deleted_at\" IS NULL AND \"providers\".\"is_enabled\" = true AND \"providers\".\"provider_vendor_id\" IS NOT NULL AND \"providers\".\"provider_vendor_id\" > 0", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "providers_provider_vendor_id_provider_vendors_id_fk": { + "name": "providers_provider_vendor_id_provider_vendors_id_fk", + "tableFrom": "providers", + "tableTo": "provider_vendors", + "columnsFrom": [ + "provider_vendor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.replay_payloads": { + "name": "replay_payloads", + "schema": "", + "columns": { + "replay_id": { + "name": "replay_id", + "type": "varchar(64)", + "primaryKey": true, + "notNull": true + }, + "verifier": { + "name": "verifier", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true + }, + "scope_tag": { + "name": "scope_tag", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true + }, + "key_id": { + "name": "key_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "format": { + "name": "format", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true + }, + "model": { + "name": "model", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false + }, + "status_code": { + "name": "status_code", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "headers_json": { + "name": "headers_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "payload": { + "name": "payload", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "byte_size": { + "name": "byte_size", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "source_message_request_id": { + "name": "source_message_request_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "idx_replay_payloads_key_id": { + "name": "idx_replay_payloads_key_id", + "columns": [ + { + "expression": "key_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_replay_payloads_expires_at": { + "name": "idx_replay_payloads_expires_at", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.request_filters": { + "name": "request_filters", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(100)", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true + }, + "action": { + "name": "action", + "type": "varchar(30)", + "primaryKey": false, + "notNull": true + }, + "match_type": { + "name": "match_type", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "target": { + "name": "target", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "replacement": { + "name": "replacement", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "priority": { + "name": "priority", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "is_enabled": { + "name": "is_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "binding_type": { + "name": "binding_type", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'global'" + }, + "provider_ids": { + "name": "provider_ids", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "group_tags": { + "name": "group_tags", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "rule_mode": { + "name": "rule_mode", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'simple'" + }, + "execution_phase": { + "name": "execution_phase", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'guard'" + }, + "operations": { + "name": "operations", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "idx_request_filters_enabled": { + "name": "idx_request_filters_enabled", + "columns": [ + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "priority", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_request_filters_scope": { + "name": "idx_request_filters_scope", + "columns": [ + { + "expression": "scope", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_request_filters_action": { + "name": "idx_request_filters_action", + "columns": [ + { + "expression": "action", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_request_filters_binding": { + "name": "idx_request_filters_binding", + "columns": [ + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "binding_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_request_filters_phase": { + "name": "idx_request_filters_phase", + "columns": [ + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "execution_phase", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sensitive_words": { + "name": "sensitive_words", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "word": { + "name": "word", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "match_type": { + "name": "match_type", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'contains'" + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_enabled": { + "name": "is_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "idx_sensitive_words_enabled": { + "name": "idx_sensitive_words_enabled", + "columns": [ + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "match_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_sensitive_words_created_at": { + "name": "idx_sensitive_words_created_at", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.system_settings": { + "name": "system_settings", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "site_title": { + "name": "site_title", + "type": "varchar(128)", + "primaryKey": false, + "notNull": true, + "default": "'CC Hub'" + }, + "allow_global_usage_view": { + "name": "allow_global_usage_view", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "currency_display": { + "name": "currency_display", + "type": "varchar(10)", + "primaryKey": false, + "notNull": true, + "default": "'USD'" + }, + "billing_model_source": { + "name": "billing_model_source", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'original'" + }, + "codex_priority_billing_source": { + "name": "codex_priority_billing_source", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'requested'" + }, + "bill_non_successful_requests": { + "name": "bill_non_successful_requests", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "bill_hedge_losers": { + "name": "bill_hedge_losers", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "discovery_enabled": { + "name": "discovery_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "discovery_concurrency": { + "name": "discovery_concurrency", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 2 + }, + "max_discovery_rounds": { + "name": "max_discovery_rounds", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 2 + }, + "discovery_sla_ms": { + "name": "discovery_sla_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 10000 + }, + "sticky_sla_ms": { + "name": "sticky_sla_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 20000 + }, + "racing_total_timeout_ms": { + "name": "racing_total_timeout_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 60000 + }, + "sticky_timeout_cooldown_ms": { + "name": "sticky_timeout_cooldown_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 300000 + }, + "timezone": { + "name": "timezone", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "enable_auto_cleanup": { + "name": "enable_auto_cleanup", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "cleanup_retention_days": { + "name": "cleanup_retention_days", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 30 + }, + "cleanup_schedule": { + "name": "cleanup_schedule", + "type": "varchar(50)", + "primaryKey": false, + "notNull": false, + "default": "'0 2 * * *'" + }, + "cleanup_batch_size": { + "name": "cleanup_batch_size", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 10000 + }, + "enable_client_version_check": { + "name": "enable_client_version_check", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "verbose_provider_error": { + "name": "verbose_provider_error", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "pass_through_upstream_error_message": { + "name": "pass_through_upstream_error_message", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "enable_http2": { + "name": "enable_http2", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "enable_openai_responses_websocket": { + "name": "enable_openai_responses_websocket", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "enable_high_concurrency_mode": { + "name": "enable_high_concurrency_mode", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "intercept_anthropic_warmup_requests": { + "name": "intercept_anthropic_warmup_requests", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "enable_thinking_signature_rectifier": { + "name": "enable_thinking_signature_rectifier", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "enable_thinking_budget_rectifier": { + "name": "enable_thinking_budget_rectifier", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "enable_thinking_effort_conflict_rectifier": { + "name": "enable_thinking_effort_conflict_rectifier", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "enable_gemini_function_id_rectifier": { + "name": "enable_gemini_function_id_rectifier", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "enable_billing_header_rectifier": { + "name": "enable_billing_header_rectifier", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "enable_response_input_rectifier": { + "name": "enable_response_input_rectifier", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "allow_non_conversation_endpoint_provider_fallback": { + "name": "allow_non_conversation_endpoint_provider_fallback", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "fake_streaming_whitelist": { + "name": "fake_streaming_whitelist", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "enable_codex_session_id_completion": { + "name": "enable_codex_session_id_completion", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "enable_claude_metadata_user_id_injection": { + "name": "enable_claude_metadata_user_id_injection", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "enable_response_fixer": { + "name": "enable_response_fixer", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "response_fixer_config": { + "name": "response_fixer_config", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'{\"fixTruncatedJson\":true,\"fixSseFormat\":true,\"fixEncoding\":true,\"maxJsonDepth\":200,\"maxFixSize\":1048576}'::jsonb" + }, + "quota_db_refresh_interval_seconds": { + "name": "quota_db_refresh_interval_seconds", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 10 + }, + "quota_lease_percent_5h": { + "name": "quota_lease_percent_5h", + "type": "numeric(5, 4)", + "primaryKey": false, + "notNull": false, + "default": "'0.05'" + }, + "quota_lease_percent_daily": { + "name": "quota_lease_percent_daily", + "type": "numeric(5, 4)", + "primaryKey": false, + "notNull": false, + "default": "'0.05'" + }, + "quota_lease_percent_weekly": { + "name": "quota_lease_percent_weekly", + "type": "numeric(5, 4)", + "primaryKey": false, + "notNull": false, + "default": "'0.05'" + }, + "quota_lease_percent_monthly": { + "name": "quota_lease_percent_monthly", + "type": "numeric(5, 4)", + "primaryKey": false, + "notNull": false, + "default": "'0.05'" + }, + "quota_lease_cap_usd": { + "name": "quota_lease_cap_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "ip_extraction_config": { + "name": "ip_extraction_config", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "ip_geo_lookup_enabled": { + "name": "ip_geo_lookup_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "public_status_window_hours": { + "name": "public_status_window_hours", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 24 + }, + "public_status_aggregation_interval_minutes": { + "name": "public_status_aggregation_interval_minutes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 5 + }, + "stream_gate_mode": { + "name": "stream_gate_mode", + "type": "varchar(10)", + "primaryKey": false, + "notNull": true, + "default": "'enforce'" + }, + "affinity_ignore_client_session_id": { + "name": "affinity_ignore_client_session_id", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "replay_enabled": { + "name": "replay_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "replay_cache_ttl_minutes": { + "name": "replay_cache_ttl_minutes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 30 + }, + "cache_effectiveness_enabled": { + "name": "cache_effectiveness_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.usage_ledger": { + "name": "usage_ledger", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "request_id": { + "name": "request_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "final_provider_id": { + "name": "final_provider_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "model": { + "name": "model", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false + }, + "original_model": { + "name": "original_model", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false + }, + "actual_response_model": { + "name": "actual_response_model", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false + }, + "endpoint": { + "name": "endpoint", + "type": "varchar(256)", + "primaryKey": false, + "notNull": false + }, + "api_type": { + "name": "api_type", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "session_id": { + "name": "session_id", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "session_identity": { + "name": "session_identity", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "session_identity_kind": { + "name": "session_identity_kind", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "affinity_scope_tag": { + "name": "affinity_scope_tag", + "type": "varchar(16)", + "primaryKey": false, + "notNull": false + }, + "affinity_fingerprint": { + "name": "affinity_fingerprint", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "affinity_fingerprint_chain": { + "name": "affinity_fingerprint_chain", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "is_replay": { + "name": "is_replay", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "replay_source_request_id": { + "name": "replay_source_request_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "status_code": { + "name": "status_code", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "is_success": { + "name": "is_success", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "success_rate_outcome": { + "name": "success_rate_outcome", + "type": "varchar(16)", + "primaryKey": false, + "notNull": false + }, + "blocked_by": { + "name": "blocked_by", + "type": "varchar(50)", + "primaryKey": false, + "notNull": false + }, + "cost_usd": { + "name": "cost_usd", + "type": "numeric(21, 15)", + "primaryKey": false, + "notNull": false, + "default": "'0'" + }, + "cost_multiplier": { + "name": "cost_multiplier", + "type": "numeric(10, 4)", + "primaryKey": false, + "notNull": false + }, + "group_cost_multiplier": { + "name": "group_cost_multiplier", + "type": "numeric(10, 4)", + "primaryKey": false, + "notNull": false + }, + "input_tokens": { + "name": "input_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "output_tokens": { + "name": "output_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "cache_creation_input_tokens": { + "name": "cache_creation_input_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "cache_read_input_tokens": { + "name": "cache_read_input_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "cache_creation_5m_input_tokens": { + "name": "cache_creation_5m_input_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "cache_creation_1h_input_tokens": { + "name": "cache_creation_1h_input_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "cache_ttl_applied": { + "name": "cache_ttl_applied", + "type": "varchar(10)", + "primaryKey": false, + "notNull": false + }, + "context_1m_applied": { + "name": "context_1m_applied", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "swap_cache_ttl_applied": { + "name": "swap_cache_ttl_applied", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "ttfb_ms": { + "name": "ttfb_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "first_byte_ms": { + "name": "first_byte_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "client_ip": { + "name": "client_ip", + "type": "varchar(45)", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "idx_usage_ledger_request_id": { + "name": "idx_usage_ledger_request_id", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_user_created_at": { + "name": "idx_usage_ledger_user_created_at", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_ledger\".\"blocked_by\" IS NULL AND \"usage_ledger\".\"is_replay\" = false", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_user_id_reset": { + "name": "idx_usage_ledger_user_id_reset", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_key_created_at": { + "name": "idx_usage_ledger_key_created_at", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_ledger\".\"blocked_by\" IS NULL AND \"usage_ledger\".\"is_replay\" = false", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_provider_created_at": { + "name": "idx_usage_ledger_provider_created_at", + "columns": [ + { + "expression": "final_provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_ledger\".\"blocked_by\" IS NULL AND \"usage_ledger\".\"is_replay\" = false", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_created_at_minute": { + "name": "idx_usage_ledger_created_at_minute", + "columns": [ + { + "expression": "date_trunc('minute', \"created_at\" AT TIME ZONE 'UTC')", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_created_at_desc_id": { + "name": "idx_usage_ledger_created_at_desc_id", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_session_id": { + "name": "idx_usage_ledger_session_id", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_ledger\".\"session_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_session_identity_created_at": { + "name": "idx_usage_ledger_session_identity_created_at", + "columns": [ + { + "expression": "COALESCE(\"session_identity\", \"session_id\")", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "\"created_at\" DESC NULLS LAST", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_ledger\".\"blocked_by\" IS NULL AND \"usage_ledger\".\"is_replay\" = false", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_session_identity": { + "name": "idx_usage_ledger_session_identity", + "columns": [ + { + "expression": "COALESCE(\"session_identity\", \"session_id\")", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_model": { + "name": "idx_usage_ledger_model", + "columns": [ + { + "expression": "model", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_ledger\".\"model\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_key_cost": { + "name": "idx_usage_ledger_key_cost", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "cost_usd", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "endpoint", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_ledger\".\"blocked_by\" IS NULL AND \"usage_ledger\".\"is_replay\" = false", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_user_cost_cover": { + "name": "idx_usage_ledger_user_cost_cover", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "cost_usd", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "endpoint", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_ledger\".\"blocked_by\" IS NULL AND \"usage_ledger\".\"is_replay\" = false", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_provider_cost_cover": { + "name": "idx_usage_ledger_provider_cost_cover", + "columns": [ + { + "expression": "final_provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "cost_usd", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "endpoint", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_ledger\".\"blocked_by\" IS NULL AND \"usage_ledger\".\"is_replay\" = false", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_key_created_at_desc_cover": { + "name": "idx_usage_ledger_key_created_at_desc_cover", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "\"created_at\" DESC NULLS LAST", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "final_provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_ledger\".\"blocked_by\" IS NULL AND \"usage_ledger\".\"is_replay\" = false", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.users": { + "name": "users", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "role": { + "name": "role", + "type": "varchar", + "primaryKey": false, + "notNull": false, + "default": "'user'" + }, + "rpm_limit": { + "name": "rpm_limit", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "daily_limit_usd": { + "name": "daily_limit_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "provider_group": { + "name": "provider_group", + "type": "varchar(200)", + "primaryKey": false, + "notNull": false, + "default": "'default'" + }, + "tags": { + "name": "tags", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'[]'::jsonb" + }, + "limit_5h_usd": { + "name": "limit_5h_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "limit_5h_reset_mode": { + "name": "limit_5h_reset_mode", + "type": "daily_reset_mode", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'rolling'" + }, + "limit_weekly_usd": { + "name": "limit_weekly_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "limit_monthly_usd": { + "name": "limit_monthly_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "limit_total_usd": { + "name": "limit_total_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "cost_reset_at": { + "name": "cost_reset_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "limit_5h_cost_reset_at": { + "name": "limit_5h_cost_reset_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "limit_concurrent_sessions": { + "name": "limit_concurrent_sessions", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "daily_reset_mode": { + "name": "daily_reset_mode", + "type": "daily_reset_mode", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'fixed'" + }, + "daily_reset_time": { + "name": "daily_reset_time", + "type": "varchar(5)", + "primaryKey": false, + "notNull": true, + "default": "'00:00'" + }, + "is_enabled": { + "name": "is_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "allowed_clients": { + "name": "allowed_clients", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'[]'::jsonb" + }, + "allowed_models": { + "name": "allowed_models", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'[]'::jsonb" + }, + "blocked_clients": { + "name": "blocked_clients", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "idx_users_active_role_sort": { + "name": "idx_users_active_role_sort", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "role", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"users\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_users_enabled_expires_at": { + "name": "idx_users_enabled_expires_at", + "columns": [ + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"users\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_users_tags_gin": { + "name": "idx_users_tags_gin", + "columns": [ + { + "expression": "tags", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"users\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "gin", + "with": {} + }, + "idx_users_created_at": { + "name": "idx_users_created_at", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_users_deleted_at": { + "name": "idx_users_deleted_at", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.webhook_targets": { + "name": "webhook_targets", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(100)", + "primaryKey": false, + "notNull": true + }, + "provider_type": { + "name": "provider_type", + "type": "webhook_provider_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "webhook_url": { + "name": "webhook_url", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": false + }, + "telegram_bot_token": { + "name": "telegram_bot_token", + "type": "varchar(256)", + "primaryKey": false, + "notNull": false + }, + "telegram_chat_id": { + "name": "telegram_chat_id", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "dingtalk_secret": { + "name": "dingtalk_secret", + "type": "varchar(256)", + "primaryKey": false, + "notNull": false + }, + "custom_template": { + "name": "custom_template", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "custom_headers": { + "name": "custom_headers", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "proxy_url": { + "name": "proxy_url", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false + }, + "proxy_fallback_to_direct": { + "name": "proxy_fallback_to_direct", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "is_enabled": { + "name": "is_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "last_test_at": { + "name": "last_test_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_test_result": { + "name": "last_test_result", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.daily_reset_mode": { + "name": "daily_reset_mode", + "schema": "public", + "values": [ + "fixed", + "rolling" + ] + }, + "public.notification_type": { + "name": "notification_type", + "schema": "public", + "values": [ + "circuit_breaker", + "daily_leaderboard", + "cost_alert", + "cache_hit_rate_alert" + ] + }, + "public.webhook_provider_type": { + "name": "webhook_provider_type", + "schema": "public", + "values": [ + "wechat", + "feishu", + "dingtalk", + "telegram", + "custom" + ] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index f09e1f133..f6b450d14 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -834,6 +834,13 @@ "when": 1785688550789, "tag": "0118_bright_sunspot", "breakpoints": true + }, + { + "idx": 119, + "version": "7", + "when": 1786038550610, + "tag": "0119_tiresome_banshee", + "breakpoints": true } ] } \ No newline at end of file diff --git a/messages/en/settings/config.json b/messages/en/settings/config.json index d75b54fc4..c6f6a1832 100644 --- a/messages/en/settings/config.json +++ b/messages/en/settings/config.json @@ -172,6 +172,10 @@ }, "replayEnabled": "Request Replay", "replayEnabledDesc": "Caches upstream responses and reuses upstream connections: identical concurrent or reconnecting requests attach to the in-flight stream instead of re-hitting the provider. Follows the ENABLE_REQUEST_REPLAY environment variable until saved here. Default on.", + "replayCacheTtlMinutes": "Replay cache duration", + "replayCacheTtlMinutesDesc": "How long completed Replay payloads remain reusable. Applies only to payloads written afterward. Range: 5-120 minutes.", + "replayCacheTtlInvalid": "Replay cache duration must be a whole number from 5 to 120 minutes.", + "replayCacheTtlMinutesUnit": "min", "cacheEffectivenessEnabled": "Prefix Cache Simulation", "cacheEffectivenessEnabledDesc": "Simulates longest-prefix cache hit rates (theoretical vs actual) for observability only; never affects routing. Follows the ENABLE_CACHE_EFFECTIVENESS environment variable until saved here. Default on." }, diff --git a/messages/ja/settings/config.json b/messages/ja/settings/config.json index 8a9aa2bae..27cb07459 100644 --- a/messages/ja/settings/config.json +++ b/messages/ja/settings/config.json @@ -172,6 +172,10 @@ }, "replayEnabled": "リクエスト Replay", "replayEnabledDesc": "上流レスポンスをキャッシュし上流接続を再利用します。同一リクエストの並行実行や再接続は進行中のストリームに追随し、プロバイダーへ再送しません。保存するまでは環境変数 ENABLE_REQUEST_REPLAY に従います。デフォルトはオン。", + "replayCacheTtlMinutes": "Replay キャッシュ時間", + "replayCacheTtlMinutesDesc": "完了した Replay payload を再利用できる時間です。以降に書き込まれるデータにのみ適用されます。範囲: 5-120 分。", + "replayCacheTtlInvalid": "Replay キャッシュ時間には 5-120 分の整数を指定してください。", + "replayCacheTtlMinutesUnit": "分", "cacheEffectivenessEnabled": "プレフィックスキャッシュシミュレーション", "cacheEffectivenessEnabledDesc": "最長プレフィックス一致のキャッシュヒット率(理論値 vs 実測値)を観測目的でシミュレートします。ルーティングには影響しません。保存するまでは環境変数 ENABLE_CACHE_EFFECTIVENESS に従います。デフォルトはオン。" }, diff --git a/messages/ru/settings/config.json b/messages/ru/settings/config.json index 3bc6d24f6..3171cce69 100644 --- a/messages/ru/settings/config.json +++ b/messages/ru/settings/config.json @@ -172,6 +172,10 @@ }, "replayEnabled": "Replay запросов", "replayEnabledDesc": "Кэширует ответы провайдера и переиспользует соединения: одинаковые параллельные или переподключающиеся запросы присоединяются к текущему потоку вместо повторного обращения к провайдеру. До сохранения следует переменной окружения ENABLE_REQUEST_REPLAY. По умолчанию включено.", + "replayCacheTtlMinutes": "Срок кэша Replay", + "replayCacheTtlMinutesDesc": "Срок повторного использования завершенных payload Replay. Применяется только к последующим записям. Диапазон: 5-120 минут.", + "replayCacheTtlInvalid": "Срок кэша Replay должен быть целым числом от 5 до 120 минут.", + "replayCacheTtlMinutesUnit": "мин", "cacheEffectivenessEnabled": "Симуляция префиксного кэша", "cacheEffectivenessEnabledDesc": "Симулирует хит-рейт кэша по наибольшему префиксу (теория и факт) только для наблюдаемости; не влияет на маршрутизацию. До сохранения следует переменной окружения ENABLE_CACHE_EFFECTIVENESS. По умолчанию включено." }, diff --git a/messages/zh-CN/settings/config.json b/messages/zh-CN/settings/config.json index 51dd007fb..6b3e4ed08 100644 --- a/messages/zh-CN/settings/config.json +++ b/messages/zh-CN/settings/config.json @@ -96,6 +96,10 @@ }, "replayEnabled": "请求 Replay", "replayEnabledDesc": "缓存上游响应并复用上游连接:并发或断线重连的相同请求直接跟尾在途流,不再重复请求供应商。保存前跟随环境变量 ENABLE_REQUEST_REPLAY,默认开启。", + "replayCacheTtlMinutes": "Replay 缓存时间", + "replayCacheTtlMinutesDesc": "已完成 Replay payload 的可重放时间,仅影响之后写入的数据。范围 5-120 分钟。", + "replayCacheTtlInvalid": "Replay 缓存时间必须是 5-120 分钟内的整数。", + "replayCacheTtlMinutesUnit": "分钟", "cacheEffectivenessEnabled": "前缀缓存模拟", "cacheEffectivenessEnabledDesc": "模拟最长前缀匹配的缓存命中率(理论 vs 实际),仅用于观测,不影响路由。保存前跟随环境变量 ENABLE_CACHE_EFFECTIVENESS,默认开启。", "affinityIgnoreClientSessionId": "忽略客户端 Session ID", diff --git a/messages/zh-TW/settings/config.json b/messages/zh-TW/settings/config.json index b233d57ce..45b9debf2 100644 --- a/messages/zh-TW/settings/config.json +++ b/messages/zh-TW/settings/config.json @@ -172,6 +172,10 @@ }, "replayEnabled": "請求 Replay", "replayEnabledDesc": "快取上游回應並重用上游連線:並發或斷線重連的相同請求直接跟尾在途串流,不再重複請求供應商。儲存前跟隨環境變數 ENABLE_REQUEST_REPLAY,預設開啟。", + "replayCacheTtlMinutes": "Replay 快取時間", + "replayCacheTtlMinutesDesc": "已完成 Replay payload 的可重播時間,僅影響之後寫入的資料。範圍 5-120 分鐘。", + "replayCacheTtlInvalid": "Replay 快取時間必須是 5-120 分鐘內的整數。", + "replayCacheTtlMinutesUnit": "分鐘", "cacheEffectivenessEnabled": "前綴快取模擬", "cacheEffectivenessEnabledDesc": "模擬最長前綴匹配的快取命中率(理論 vs 實際),僅用於觀測,不影響路由。儲存前跟隨環境變數 ENABLE_CACHE_EFFECTIVENESS,預設開啟。" }, diff --git a/src/actions/system-config.ts b/src/actions/system-config.ts index 639f6dbe0..d673e45fa 100644 --- a/src/actions/system-config.ts +++ b/src/actions/system-config.ts @@ -20,6 +20,10 @@ import { DISCOVERY_WINDOW_INVALID_ERROR_CODE, getDiscoveryValidationErrorCode, } from "@/lib/validation/discovery-settings"; +import { + getReplayCacheTtlValidationErrorCode, + REPLAY_CACHE_TTL_INVALID_ERROR_CODE, +} from "@/lib/validation/replay-settings"; import { UpdateSystemSettingsSchema } from "@/lib/validation/schemas"; import { getSystemSettings, updateSystemSettings } from "@/repository/system-config"; import type { IpExtractionConfig } from "@/types/ip-extraction"; @@ -32,9 +36,13 @@ import type { } from "@/types/system-config"; import type { ActionResult } from "./types"; -function discoveryValidationErrorCode(error: unknown): string | null { +function systemSettingsValidationErrorCode(error: unknown): string | null { if (!(error instanceof ZodError)) return null; - return getDiscoveryValidationErrorCode(error.issues) ?? null; + return ( + getDiscoveryValidationErrorCode(error.issues) ?? + getReplayCacheTtlValidationErrorCode(error.issues) ?? + null + ); } export async function fetchSystemSettings(): Promise> { @@ -106,6 +114,7 @@ export async function saveSystemSettings(formData: { streamGateMode?: StreamGateSettingMode; affinityIgnoreClientSessionId?: boolean; replayEnabled?: boolean | null; + replayCacheTtlMinutes?: number; cacheEffectivenessEnabled?: boolean | null; enableCodexSessionIdCompletion?: boolean; enableClaudeMetadataUserIdInjection?: boolean; @@ -196,6 +205,7 @@ export async function saveSystemSettings(formData: { streamGateMode: validated.streamGateMode, affinityIgnoreClientSessionId: validated.affinityIgnoreClientSessionId, replayEnabled: validated.replayEnabled, + replayCacheTtlMinutes: validated.replayCacheTtlMinutes, cacheEffectivenessEnabled: validated.cacheEffectivenessEnabled, enableCodexSessionIdCompletion: validated.enableCodexSessionIdCompletion, enableClaudeMetadataUserIdInjection: validated.enableClaudeMetadataUserIdInjection, @@ -297,9 +307,11 @@ export async function saveSystemSettings(formData: { return { ok: true, data: { ...updated, publicStatusProjectionWarningCode } }; } catch (error) { logger.error("更新系统设置失败:", error); - const validationErrorCode = discoveryValidationErrorCode(error); + const validationErrorCode = systemSettingsValidationErrorCode(error); const message = validationErrorCode - ? "Discovery settings validation failed." + ? validationErrorCode === REPLAY_CACHE_TTL_INVALID_ERROR_CODE + ? "Replay cache TTL validation failed." + : "Discovery settings validation failed." : error instanceof Error ? error.message : "更新系统设置失败"; diff --git a/src/app/[locale]/settings/config/_components/system-settings-form.tsx b/src/app/[locale]/settings/config/_components/system-settings-form.tsx index 8a371b85c..789c01b02 100644 --- a/src/app/[locale]/settings/config/_components/system-settings-form.tsx +++ b/src/app/[locale]/settings/config/_components/system-settings-form.tsx @@ -52,6 +52,11 @@ import { shouldWarnQuotaLeasePercentZero, } from "@/lib/utils/validation/quota-lease-warnings"; import { DISCOVERY_FIELD_LIMITS } from "@/lib/validation/discovery-settings"; +import { + REPLAY_CACHE_TTL_INVALID_ERROR_CODE, + REPLAY_CACHE_TTL_MINUTES_MAX, + REPLAY_CACHE_TTL_MINUTES_MIN, +} from "@/lib/validation/replay-settings"; import { DEFAULT_IP_EXTRACTION_CONFIG, type IpExtractionConfig } from "@/types/ip-extraction"; import type { BillingModelSource, @@ -100,6 +105,7 @@ interface SystemSettingsFormProps { | "streamGateMode" | "affinityIgnoreClientSessionId" | "replayEnabled" + | "replayCacheTtlMinutes" | "cacheEffectivenessEnabled" | "enableCodexSessionIdCompletion" | "enableClaudeMetadataUserIdInjection" @@ -220,6 +226,9 @@ export function SystemSettingsForm({ // null = 跟随环境变量：未触碰开关时按 null 原样保存， // 避免无关字段的保存把覆写写死为布尔值；仅用户切换后才落显式值 const [replayEnabled, setReplayEnabled] = useState(initialSettings.replayEnabled); + const [replayCacheTtlMinutes, setReplayCacheTtlMinutes] = useState( + String(initialSettings.replayCacheTtlMinutes) + ); const [cacheEffectivenessEnabled, setCacheEffectivenessEnabled] = useState( initialSettings.cacheEffectivenessEnabled ); @@ -409,6 +418,7 @@ export function SystemSettingsForm({ streamGateMode, affinityIgnoreClientSessionId, replayEnabled, + replayCacheTtlMinutes: Number(replayCacheTtlMinutes), cacheEffectivenessEnabled, enableThinkingBudgetRectifier, enableThinkingEffortConflictRectifier, @@ -433,7 +443,9 @@ export function SystemSettingsForm({ ? t("discoveryWindowInvalid") : result.errorCode === "DISCOVERY_SETTINGS_INVALID" ? t("discoverySettingsInvalid") - : result.error || t("saveFailed"); + : result.errorCode === REPLAY_CACHE_TTL_INVALID_ERROR_CODE + ? t("replayCacheTtlInvalid") + : result.error || t("saveFailed"); toast.error(errorMessage); return; } @@ -475,6 +487,7 @@ export function SystemSettingsForm({ setStreamGateMode(result.data.streamGateMode); setAffinityIgnoreClientSessionId(result.data.affinityIgnoreClientSessionId); setReplayEnabled(result.data.replayEnabled ?? null); + setReplayCacheTtlMinutes(String(result.data.replayCacheTtlMinutes)); setCacheEffectivenessEnabled(result.data.cacheEffectivenessEnabled ?? null); setEnableThinkingBudgetRectifier(result.data.enableThinkingBudgetRectifier); setEnableThinkingEffortConflictRectifier(result.data.enableThinkingEffortConflictRectifier); @@ -743,11 +756,11 @@ export function SystemSettingsForm({ {/* Bounded Streaming Discovery */}
-
+
-
+

{t("discoveryEnabled")}

{t("discoveryEnabledDesc")}

@@ -1177,23 +1190,51 @@ export function SystemSettingsForm({
{/* F2 Request Replay */} -
-
-
- +
+
+
+
+ +
+
+

{t("replayEnabled")}

+

{t("replayEnabledDesc")}

+
+ setReplayEnabled(checked)} + disabled={isPending} + /> +
+
-

{t("replayEnabled")}

-

{t("replayEnabledDesc")}

+ +

+ {t("replayCacheTtlMinutesDesc")} +

+
+
+ setReplayCacheTtlMinutes(event.target.value)} + disabled={isPending} + className={`${inputClassName} max-sm:text-base`} + /> +

+ {t("replayCacheTtlMinutesUnit")} +

- setReplayEnabled(checked)} - disabled={isPending} - />
{/* F3b Cache Effectiveness Simulation */} diff --git a/src/app/[locale]/settings/config/page.tsx b/src/app/[locale]/settings/config/page.tsx index c66ae4e87..c32f4e579 100644 --- a/src/app/[locale]/settings/config/page.tsx +++ b/src/app/[locale]/settings/config/page.tsx @@ -83,6 +83,7 @@ async function SettingsConfigContent({ locale }: { locale: string }) { streamGateMode: settings.streamGateMode, affinityIgnoreClientSessionId: settings.affinityIgnoreClientSessionId, replayEnabled: settings.replayEnabled, + replayCacheTtlMinutes: settings.replayCacheTtlMinutes, cacheEffectivenessEnabled: settings.cacheEffectivenessEnabled, enableCodexSessionIdCompletion: settings.enableCodexSessionIdCompletion, enableClaudeMetadataUserIdInjection: settings.enableClaudeMetadataUserIdInjection, diff --git a/src/app/api/v1/resources/system/router.ts b/src/app/api/v1/resources/system/router.ts index 3e693e98f..b3962aa50 100644 --- a/src/app/api/v1/resources/system/router.ts +++ b/src/app/api/v1/resources/system/router.ts @@ -10,6 +10,7 @@ import { SystemTimezoneResponseSchema, } from "@/lib/api/v1/schemas/system-config"; import { getDiscoveryValidationErrorCode } from "@/lib/validation/discovery-settings"; +import { getReplayCacheTtlValidationErrorCode } from "@/lib/validation/replay-settings"; import { getSystemDisplaySettings, getSystemSettings, @@ -23,7 +24,8 @@ export const systemRouter = new OpenAPIHono({ return fromZodError( result.error, new URL(c.req.url).pathname, - getDiscoveryValidationErrorCode(result.error.issues) + getDiscoveryValidationErrorCode(result.error.issues) ?? + getReplayCacheTtlValidationErrorCode(result.error.issues) ); } }, diff --git a/src/app/v1/_lib/proxy/replay/replay-store.ts b/src/app/v1/_lib/proxy/replay/replay-store.ts index a58875a96..02a6c44fb 100644 --- a/src/app/v1/_lib/proxy/replay/replay-store.ts +++ b/src/app/v1/_lib/proxy/replay/replay-store.ts @@ -9,6 +9,12 @@ import { logger } from "@/lib/logger"; import { getRedisClient } from "@/lib/redis/client"; import { RedisKVStore } from "@/lib/redis/redis-kv-store"; import { RedisListStore } from "@/lib/redis/redis-list-store"; +import { getCachedProxyRuntimeSettings } from "@/lib/system-settings/proxy-runtime"; +import { + REPLAY_CACHE_TTL_MINUTES_DEFAULT, + REPLAY_CACHE_TTL_MINUTES_MAX, + REPLAY_CACHE_TTL_MINUTES_MIN, +} from "@/lib/validation/replay-settings"; /** * F2 Replay 双层存储： @@ -373,9 +379,8 @@ export class ReplayStore { * （过期行清理由 instrumentation 定时调度器负责，不在写路径顺带执行。） */ async persistCompleted(row: ReplayPersistedRow): Promise<"persisted" | "existing"> { - const env = getEnvConfig(); const now = new Date(); - const expiresAt = new Date(now.getTime() + env.REPLAY_COMPLETED_TTL_SECONDS * 1000); + const expiresAt = new Date(now.getTime() + resolveReplayCompletedTtlSeconds() * 1000); const persistedValues = { verifier: row.verifier, scopeTag: row.scopeTag, @@ -434,7 +439,7 @@ export class ReplayStore { WITH doomed AS ( SELECT replay_id FROM replay_payloads - WHERE expires_at < ${cutoff} + WHERE expires_at < ${sql.param(cutoff, replayPayloads.expiresAt)} ORDER BY expires_at, replay_id LIMIT ${REPLAY_CLEANUP_BATCH_SIZE} FOR UPDATE SKIP LOCKED @@ -466,13 +471,25 @@ export class ReplayStore { } export function resolveReplayTtlSeconds(): number { + const completedTtlSeconds = resolveReplayCompletedTtlSeconds(); try { - return getEnvConfig().REPLAY_TTL_SECONDS; + return Math.min(getEnvConfig().REPLAY_TTL_SECONDS, completedTtlSeconds); } catch { - return 600; + return Math.min(600, completedTtlSeconds); } } +export function resolveReplayCompletedTtlSeconds(): number { + const minutes = + getCachedProxyRuntimeSettings()?.replayCacheTtlMinutes ?? REPLAY_CACHE_TTL_MINUTES_DEFAULT; + return ( + Math.max( + REPLAY_CACHE_TTL_MINUTES_MIN, + Math.min(REPLAY_CACHE_TTL_MINUTES_MAX, Math.trunc(minutes)) + ) * 60 + ); +} + let sharedReplayStore: ReplayStore | null = null; export function getReplayStore(): ReplayStore { diff --git a/src/drizzle/schema.ts b/src/drizzle/schema.ts index f0684ff8e..60f497c0a 100644 --- a/src/drizzle/schema.ts +++ b/src/drizzle/schema.ts @@ -1056,6 +1056,8 @@ export const systemSettings = pgTable('system_settings', { // F2 Replay 开关覆写（null = 跟随环境变量 ENABLE_REQUEST_REPLAY） replayEnabled: boolean('replay_enabled'), + // F2 Replay 完成 payload 的可重放窗口(分钟,默认 30) + replayCacheTtlMinutes: integer('replay_cache_ttl_minutes').notNull().default(30), // F3b 最长前缀匹配缓存模拟开关覆写（null = 跟随环境变量 ENABLE_CACHE_EFFECTIVENESS） cacheEffectivenessEnabled: boolean('cache_effectiveness_enabled'), diff --git a/src/instrumentation.ts b/src/instrumentation.ts index 192eb0ac1..45b0b23b6 100644 --- a/src/instrumentation.ts +++ b/src/instrumentation.ts @@ -328,11 +328,17 @@ export async function startReplayCleanupScheduler(): Promise { const intervalMs = 10 * 60 * 1000; const runTick = () => { - void runReplayCleanupTick().catch((error) => { - logger.warn("[Instrumentation] Replay cleanup tick failed", { - error: error instanceof Error ? error.message : String(error), + void runReplayCleanupTick() + .then((result) => { + if (result.deleted > 0) { + logger.info("[Instrumentation] Replay cleanup tick completed", result); + } + }) + .catch((error) => { + logger.warn("[Instrumentation] Replay cleanup tick failed", { + ...describeSchedulerError(error), + }); }); - }); }; runTick(); @@ -344,7 +350,7 @@ export async function startReplayCleanupScheduler(): Promise { }); } catch (error) { logger.warn("[Instrumentation] Replay cleanup scheduler init failed", { - error: error instanceof Error ? error.message : String(error), + ...describeSchedulerError(error), }); } } diff --git a/src/lib/api-client/v1/openapi-types.gen.ts b/src/lib/api-client/v1/openapi-types.gen.ts index 82c1038ef..01539443a 100644 --- a/src/lib/api-client/v1/openapi-types.gen.ts +++ b/src/lib/api-client/v1/openapi-types.gen.ts @@ -12262,6 +12262,8 @@ export interface operations { affinityIgnoreClientSessionId: boolean; /** @description Request replay (response caching and upstream connection reuse) override. Null follows the ENABLE_REQUEST_REPLAY environment variable. */ replayEnabled: boolean | null; + /** @description Replay completed payload reuse window in minutes. */ + replayCacheTtlMinutes: number; /** @description Longest-prefix cache-effectiveness simulation override (observability only). Null follows the ENABLE_CACHE_EFFECTIVENESS environment variable. */ cacheEffectivenessEnabled: boolean | null; /** @@ -12548,6 +12550,8 @@ export interface operations { affinityIgnoreClientSessionId?: boolean; /** @description Request replay (response caching and upstream connection reuse) override. Null follows the ENABLE_REQUEST_REPLAY environment variable. */ replayEnabled?: boolean | null; + /** @description Replay completed payload reuse window in minutes. */ + replayCacheTtlMinutes?: number; /** @description Longest-prefix cache-effectiveness simulation override (observability only). Null follows the ENABLE_CACHE_EFFECTIVENESS environment variable. */ cacheEffectivenessEnabled?: boolean | null; }; @@ -12709,6 +12713,8 @@ export interface operations { affinityIgnoreClientSessionId: boolean; /** @description Request replay (response caching and upstream connection reuse) override. Null follows the ENABLE_REQUEST_REPLAY environment variable. */ replayEnabled: boolean | null; + /** @description Replay completed payload reuse window in minutes. */ + replayCacheTtlMinutes: number; /** @description Longest-prefix cache-effectiveness simulation override (observability only). Null follows the ENABLE_CACHE_EFFECTIVENESS environment variable. */ cacheEffectivenessEnabled: boolean | null; /** diff --git a/src/lib/api/v1/schemas/system-config.ts b/src/lib/api/v1/schemas/system-config.ts index c3c06dd70..5dd0253a6 100644 --- a/src/lib/api/v1/schemas/system-config.ts +++ b/src/lib/api/v1/schemas/system-config.ts @@ -4,6 +4,10 @@ import { DISCOVERY_FIELD_LIMITS, DISCOVERY_SETTINGS_INVALID_ERROR_CODE, } from "@/lib/validation/discovery-settings"; +import { + REPLAY_CACHE_TTL_MINUTES_MAX, + REPLAY_CACHE_TTL_MINUTES_MIN, +} from "@/lib/validation/replay-settings"; import { IsoDateTimeStringSchema } from "./_common"; const currencyValues = Object.keys(CURRENCY_CONFIG) as [ @@ -228,6 +232,12 @@ export const SystemSettingsSchema = z .describe( "Request replay (response caching and upstream connection reuse) override. Null follows the ENABLE_REQUEST_REPLAY environment variable." ), + replayCacheTtlMinutes: z + .number() + .int() + .min(REPLAY_CACHE_TTL_MINUTES_MIN) + .max(REPLAY_CACHE_TTL_MINUTES_MAX) + .describe("Replay completed payload reuse window in minutes."), cacheEffectivenessEnabled: z .boolean() .nullable() diff --git a/src/lib/config/env.schema.ts b/src/lib/config/env.schema.ts index f0bcdff90..7193f9549 100644 --- a/src/lib/config/env.schema.ts +++ b/src/lib/config/env.schema.ts @@ -205,8 +205,6 @@ export const EnvSchema = z.object({ REPLAY_MAX_CONCURRENT_SPOOLS: z.coerce.number().int().min(1).max(1024).default(64), // Redis 热层 TTL（活跃/刚完成的响应块与元数据） REPLAY_TTL_SECONDS: z.coerce.number().int().min(60).max(7200).default(600), - // PG 完成持久层 TTL（跨小时级重放窗口） - REPLAY_COMPLETED_TTL_SECONDS: z.coerce.number().int().min(300).max(86400).default(3600), // 单响应缓存上限（超限即放弃 spool，fail-open 回现状） REPLAY_MAX_PAYLOAD_BYTES: z.coerce .number() diff --git a/src/lib/config/system-settings-cache.ts b/src/lib/config/system-settings-cache.ts index 8f8cc0511..ec5032b9f 100644 --- a/src/lib/config/system-settings-cache.ts +++ b/src/lib/config/system-settings-cache.ts @@ -14,6 +14,7 @@ import { logger } from "@/lib/logger"; import { DEFAULT_SITE_TITLE } from "@/lib/site-title"; +import { REPLAY_CACHE_TTL_MINUTES_DEFAULT } from "@/lib/validation/replay-settings"; import { getSystemSettings } from "@/repository/system-config"; import type { SystemSettings } from "@/types/system-config"; import { getEnvConfig } from "./env.schema"; @@ -108,6 +109,7 @@ export const DEFAULT_SETTINGS: Pick< | "publicStatusAggregationIntervalMinutes" | "streamGateMode" | "affinityIgnoreClientSessionId" + | "replayCacheTtlMinutes" | "discoveryEnabled" | "discoveryConcurrency" | "maxDiscoveryRounds" @@ -147,6 +149,7 @@ export const DEFAULT_SETTINGS: Pick< publicStatusAggregationIntervalMinutes: 5, streamGateMode: "enforce", affinityIgnoreClientSessionId: true, + replayCacheTtlMinutes: REPLAY_CACHE_TTL_MINUTES_DEFAULT, discoveryEnabled: false, discoveryConcurrency: 2, maxDiscoveryRounds: 2, @@ -239,6 +242,7 @@ export async function getCachedSystemSettings(): Promise { streamGateMode: getFallbackStreamGateMode(), affinityIgnoreClientSessionId: DEFAULT_SETTINGS.affinityIgnoreClientSessionId, replayEnabled: null, + replayCacheTtlMinutes: DEFAULT_SETTINGS.replayCacheTtlMinutes, cacheEffectivenessEnabled: null, discoveryEnabled: DEFAULT_SETTINGS.discoveryEnabled, discoveryConcurrency: DEFAULT_SETTINGS.discoveryConcurrency, diff --git a/src/lib/system-settings/proxy-runtime.ts b/src/lib/system-settings/proxy-runtime.ts index 94d1a5604..48c17e3d6 100644 --- a/src/lib/system-settings/proxy-runtime.ts +++ b/src/lib/system-settings/proxy-runtime.ts @@ -2,6 +2,7 @@ import "server-only"; import { getEnvConfig } from "@/lib/config/env.schema"; import { getCachedSystemSettings } from "@/lib/config/system-settings-cache"; +import { REPLAY_CACHE_TTL_MINUTES_DEFAULT } from "@/lib/validation/replay-settings"; /** * 代理热路径消费的系统设置快照。 @@ -20,6 +21,7 @@ export interface ProxyRuntimeSettings { streamGateMode: "off" | "shadow" | "enforce"; affinityIgnoreClientSessionId: boolean; replayEnabled: boolean; + replayCacheTtlMinutes: number; cacheEffectivenessEnabled: boolean; } @@ -49,6 +51,7 @@ function envFallback(): ProxyRuntimeSettings { streamGateMode: env.STREAM_GATE_MODE, affinityIgnoreClientSessionId: true, replayEnabled: env.ENABLE_REQUEST_REPLAY, + replayCacheTtlMinutes: REPLAY_CACHE_TTL_MINUTES_DEFAULT, cacheEffectivenessEnabled: env.ENABLE_CACHE_EFFECTIVENESS, }; } catch { @@ -56,6 +59,7 @@ function envFallback(): ProxyRuntimeSettings { streamGateMode: "off", affinityIgnoreClientSessionId: true, replayEnabled: false, + replayCacheTtlMinutes: REPLAY_CACHE_TTL_MINUTES_DEFAULT, cacheEffectivenessEnabled: true, }; } @@ -68,6 +72,7 @@ export async function getProxyRuntimeSettings(): Promise { streamGateMode: settings.streamGateMode, affinityIgnoreClientSessionId: settings.affinityIgnoreClientSessionId, replayEnabled: settings.replayEnabled ?? envReplayDefault(), + replayCacheTtlMinutes: settings.replayCacheTtlMinutes ?? REPLAY_CACHE_TTL_MINUTES_DEFAULT, cacheEffectivenessEnabled: settings.cacheEffectivenessEnabled ?? envCacheEffectivenessDefault(), }; diff --git a/src/lib/validation/replay-settings.ts b/src/lib/validation/replay-settings.ts new file mode 100644 index 000000000..c625e6c15 --- /dev/null +++ b/src/lib/validation/replay-settings.ts @@ -0,0 +1,16 @@ +export const REPLAY_CACHE_TTL_MINUTES_DEFAULT = 30; +export const REPLAY_CACHE_TTL_MINUTES_MIN = 5; +export const REPLAY_CACHE_TTL_MINUTES_MAX = 120; +export const REPLAY_CACHE_TTL_INVALID_ERROR_CODE = "REPLAY_CACHE_TTL_INVALID"; + +export function getReplayCacheTtlValidationErrorCode( + issues: ReadonlyArray<{ message: string; path: readonly PropertyKey[] }> +): string | undefined { + return issues.some( + (issue) => + issue.path[0] === "replayCacheTtlMinutes" || + issue.message === REPLAY_CACHE_TTL_INVALID_ERROR_CODE + ) + ? REPLAY_CACHE_TTL_INVALID_ERROR_CODE + : undefined; +} diff --git a/src/lib/validation/schemas.ts b/src/lib/validation/schemas.ts index c5bba2c71..58c259857 100644 --- a/src/lib/validation/schemas.ts +++ b/src/lib/validation/schemas.ts @@ -21,6 +21,11 @@ import { DISCOVERY_SETTINGS_INVALID_ERROR_CODE, DISCOVERY_WINDOW_INVALID_ERROR_CODE, } from "./discovery-settings"; +import { + REPLAY_CACHE_TTL_INVALID_ERROR_CODE, + REPLAY_CACHE_TTL_MINUTES_MAX, + REPLAY_CACHE_TTL_MINUTES_MIN, +} from "./replay-settings"; export { DISCOVERY_SETTINGS_INVALID_ERROR_CODE, @@ -1120,6 +1125,13 @@ export const UpdateSystemSettingsSchema = z affinityIgnoreClientSessionId: z.boolean().optional(), // F2 Replay 响应缓存与复用（可选；null = 跟随环境变量） replayEnabled: z.boolean().nullable().optional(), + // F2 Replay 完成 payload 可重放窗口(分钟) + replayCacheTtlMinutes: z.coerce + .number() + .int(REPLAY_CACHE_TTL_INVALID_ERROR_CODE) + .min(REPLAY_CACHE_TTL_MINUTES_MIN, REPLAY_CACHE_TTL_INVALID_ERROR_CODE) + .max(REPLAY_CACHE_TTL_MINUTES_MAX, REPLAY_CACHE_TTL_INVALID_ERROR_CODE) + .optional(), // F3b 最长前缀匹配缓存模拟（可选；null = 跟随环境变量） cacheEffectivenessEnabled: z.boolean().nullable().optional(), // Codex Session ID 补全（可选） diff --git a/src/repository/_shared/transformers.test.ts b/src/repository/_shared/transformers.test.ts index f80c37498..b9c515058 100644 --- a/src/repository/_shared/transformers.test.ts +++ b/src/repository/_shared/transformers.test.ts @@ -291,10 +291,15 @@ describe("src/repository/_shared/transformers.ts", () => { expect(result.enableHttp2).toBe(false); expect(result.enableOpenaiResponsesWebsocket).toBe(true); expect(result.interceptAnthropicWarmupRequests).toBe(false); + expect(result.replayCacheTtlMinutes).toBe(30); expect(result.createdAt).toEqual(now); expect(result.updatedAt).toEqual(now); }); + it("应映射 Replay 缓存时间", () => { + expect(toSystemSettings({ replayCacheTtlMinutes: 45 }).replayCacheTtlMinutes).toBe(45); + }); + it("应映射 interceptAnthropicWarmupRequests 字段", () => { const result = toSystemSettings({ id: 1, diff --git a/src/repository/_shared/transformers.ts b/src/repository/_shared/transformers.ts index 474d19b8d..f401d6485 100644 --- a/src/repository/_shared/transformers.ts +++ b/src/repository/_shared/transformers.ts @@ -2,6 +2,7 @@ import { PROVIDER_TIMEOUT_DEFAULTS } from "@/lib/constants/provider.constants"; import { normalizeProviderModelRedirectRules } from "@/lib/provider-model-redirects"; import { DEFAULT_SITE_TITLE } from "@/lib/site-title"; import { formatCostForStorage } from "@/lib/utils/currency"; +import { REPLAY_CACHE_TTL_MINUTES_DEFAULT } from "@/lib/validation/replay-settings"; import type { Key } from "@/types/key"; import type { MessageRequest } from "@/types/message"; import type { ModelPrice } from "@/types/model-price"; @@ -320,6 +321,7 @@ export function toSystemSettings(dbSettings: any): SystemSettings { : "enforce", affinityIgnoreClientSessionId: dbSettings?.affinityIgnoreClientSessionId ?? true, replayEnabled: dbSettings?.replayEnabled ?? null, + replayCacheTtlMinutes: dbSettings?.replayCacheTtlMinutes ?? REPLAY_CACHE_TTL_MINUTES_DEFAULT, cacheEffectivenessEnabled: dbSettings?.cacheEffectivenessEnabled ?? null, createdAt: dbSettings?.createdAt ? new Date(dbSettings.createdAt) : new Date(), updatedAt: dbSettings?.updatedAt ? new Date(dbSettings.updatedAt) : new Date(), diff --git a/src/repository/system-config.ts b/src/repository/system-config.ts index ae2a0eb1d..c26d8fcba 100644 --- a/src/repository/system-config.ts +++ b/src/repository/system-config.ts @@ -6,6 +6,7 @@ import { db } from "@/drizzle/db"; import { systemSettings } from "@/drizzle/schema"; import { logger } from "@/lib/logger"; import { DEFAULT_SITE_TITLE } from "@/lib/site-title"; +import { REPLAY_CACHE_TTL_MINUTES_DEFAULT } from "@/lib/validation/replay-settings"; import { DEFAULT_FAKE_STREAMING_WHITELIST, type SystemSettings, @@ -203,6 +204,7 @@ function createFallbackSettings(): SystemSettings { streamGateMode: "enforce", affinityIgnoreClientSessionId: true, replayEnabled: null, + replayCacheTtlMinutes: REPLAY_CACHE_TTL_MINUTES_DEFAULT, cacheEffectivenessEnabled: null, createdAt: now, updatedAt: now, @@ -282,6 +284,12 @@ const RECENT_COLUMN_LADDER: ReadonlyArray<{ // 本层更新失败（仍有列缺失）时记录的告警 updateWarn: string; }> = [ + { + key: "replayCacheTtlMinutes", + column: systemSettings.replayCacheTtlMinutes, + selectWarn: "system_settings 缺少 replayCacheTtlMinutes,回退到上一代字段集.", + updateWarn: "system_settings 缺少 replayCacheTtlMinutes,继续降级更新.", + }, { key: "cacheEffectivenessEnabled", column: systemSettings.cacheEffectivenessEnabled, @@ -554,6 +562,7 @@ export async function getSystemSettings(): Promise { enableHighConcurrencyMode: false, publicStatusWindowHours: 24, publicStatusAggregationIntervalMinutes: 5, + replayCacheTtlMinutes: REPLAY_CACHE_TTL_MINUTES_DEFAULT, }) .onConflictDoNothing(); } catch (error) { @@ -896,6 +905,11 @@ export async function updateSystemSettings( updates.replayEnabled = payload.replayEnabled; } + // F2 Replay 完成 payload 可重放窗口(分钟) + if (payload.replayCacheTtlMinutes !== undefined) { + updates.replayCacheTtlMinutes = payload.replayCacheTtlMinutes; + } + // F3b 缓存模拟开关覆写（如果提供；null = 清除覆写跟随环境变量） if (payload.cacheEffectivenessEnabled !== undefined) { updates.cacheEffectivenessEnabled = payload.cacheEffectivenessEnabled; diff --git a/src/types/system-config.ts b/src/types/system-config.ts index 401642f5d..a1eea2b3a 100644 --- a/src/types/system-config.ts +++ b/src/types/system-config.ts @@ -161,6 +161,8 @@ export interface SystemSettings { // F2 Replay（响应缓存与上游连接复用）开关覆写 // null = 跟随环境变量 ENABLE_REQUEST_REPLAY（默认 true） replayEnabled: boolean | null; + // F2 Replay 完成 payload 的可重放窗口(分钟) + replayCacheTtlMinutes: number; // F3b 最长前缀匹配缓存模拟（理论 vs 实际缓存命中率，仅观测不影响路由）开关覆写 // null = 跟随环境变量 ENABLE_CACHE_EFFECTIVENESS（默认 true） @@ -295,6 +297,8 @@ export interface UpdateSystemSettingsInput { // F2 Replay 开关（可选；null = 清除覆写跟随环境变量） replayEnabled?: boolean | null; + // F2 Replay 完成 payload 的可重放窗口(分钟) + replayCacheTtlMinutes?: number; // F3b 缓存模拟开关（可选；null = 清除覆写跟随环境变量） cacheEffectivenessEnabled?: boolean | null; diff --git a/tests/api/v1/system/system-config.test.ts b/tests/api/v1/system/system-config.test.ts index 2409e9e3b..c16d0eac6 100644 --- a/tests/api/v1/system/system-config.test.ts +++ b/tests/api/v1/system/system-config.test.ts @@ -78,6 +78,7 @@ const settings: SystemSettings = { quotaLeaseCapUsd: null, publicStatusWindowHours: 24, publicStatusAggregationIntervalMinutes: 5, + replayCacheTtlMinutes: 30, ipExtractionConfig: null, ipGeoLookupEnabled: true, createdAt: new Date("2026-04-28T00:00:00.000Z"), @@ -92,7 +93,7 @@ describe("v1 system config endpoints", () => { getSystemSettingsRepoMock.mockResolvedValue(settings); saveSystemSettingsMock.mockResolvedValue({ ok: true, - data: { ...settings, siteTitle: "CCH Ops", timezone: "UTC" }, + data: { ...settings, siteTitle: "CCH Ops", timezone: "UTC", replayCacheTtlMinutes: 45 }, }); getServerTimeZoneMock.mockResolvedValue({ ok: true, data: { timeZone: "Asia/Shanghai" } }); }); @@ -113,13 +114,18 @@ describe("v1 system config endpoints", () => { method: "PUT", pathname: "/api/v1/system/settings", headers: { Authorization: "Bearer admin-token" }, - body: { siteTitle: "CCH Ops", timezone: "UTC" }, + body: { siteTitle: "CCH Ops", timezone: "UTC", replayCacheTtlMinutes: 45 }, }); expect(updated.response.status).toBe(200); - expect(updated.json).toMatchObject({ siteTitle: "CCH Ops", timezone: "UTC" }); + expect(updated.json).toMatchObject({ + siteTitle: "CCH Ops", + timezone: "UTC", + replayCacheTtlMinutes: 45, + }); expect(saveSystemSettingsMock).toHaveBeenCalledWith({ siteTitle: "CCH Ops", timezone: "UTC", + replayCacheTtlMinutes: 45, }); }); @@ -174,6 +180,15 @@ describe("v1 system config endpoints", () => { }); expect(invalidTimezone.response.status).toBe(400); expect(invalidTimezone.json).toMatchObject({ errorCode: "request.validation_failed" }); + + const invalidReplayTtl = await callV1Route({ + method: "PUT", + pathname: "/api/v1/system/settings", + headers: { Authorization: "Bearer admin-token" }, + body: { replayCacheTtlMinutes: 4 }, + }); + expect(invalidReplayTtl.response.status).toBe(400); + expect(invalidReplayTtl.json).toMatchObject({ errorCode: "REPLAY_CACHE_TTL_INVALID" }); }); test("returns a stable error code for out-of-range Discovery settings", async () => { diff --git a/tests/unit/actions/system-config-save.test.ts b/tests/unit/actions/system-config-save.test.ts index 89c19dd30..ebc5cc160 100644 --- a/tests/unit/actions/system-config-save.test.ts +++ b/tests/unit/actions/system-config-save.test.ts @@ -152,6 +152,26 @@ describe("saveSystemSettings", () => { ); }); + it("accepts and forwards the Replay cache TTL", async () => { + const result = await saveSystemSettings({ replayCacheTtlMinutes: 30 }); + + expect(result.ok).toBe(true); + expect(updateSystemSettingsMock).toHaveBeenCalledWith( + expect.objectContaining({ replayCacheTtlMinutes: 30 }) + ); + }); + + it.each([4, 121, 30.5])("rejects invalid Replay cache TTL %s", async (value) => { + const result = await saveSystemSettings({ replayCacheTtlMinutes: value }); + + expect(result).toMatchObject({ + ok: false, + error: "Replay cache TTL validation failed.", + errorCode: "REPLAY_CACHE_TTL_INVALID", + }); + expect(updateSystemSettingsMock).not.toHaveBeenCalled(); + }); + it("returns a structured error code for invalid Discovery field ranges", async () => { const result = await saveSystemSettings({ discoveryConcurrency: 33 }); diff --git a/tests/unit/instrumentation-replay-cleanup.test.ts b/tests/unit/instrumentation-replay-cleanup.test.ts index 54bd81e77..937aca87a 100644 --- a/tests/unit/instrumentation-replay-cleanup.test.ts +++ b/tests/unit/instrumentation-replay-cleanup.test.ts @@ -4,11 +4,16 @@ const cleanupControl = vi.hoisted(() => ({ runReplayCleanupTick: vi.fn(), })); +const loggerControl = vi.hoisted(() => ({ + info: vi.fn(), + warn: vi.fn(), +})); + vi.mock("@/lib/logger", () => ({ logger: { debug: vi.fn(), - info: vi.fn(), - warn: vi.fn(), + info: loggerControl.info, + warn: loggerControl.warn, error: vi.fn(), trace: vi.fn(), fatal: vi.fn(), @@ -24,6 +29,8 @@ import { startReplayCleanupScheduler } from "@/instrumentation"; describe("startReplayCleanupScheduler", () => { beforeEach(() => { vi.useFakeTimers(); + loggerControl.info.mockReset(); + loggerControl.warn.mockReset(); cleanupControl.runReplayCleanupTick.mockReset().mockResolvedValue({ status: "completed", batches: 1, @@ -55,4 +62,41 @@ describe("startReplayCleanupScheduler", () => { expect(setIntervalSpy).toHaveBeenCalledWith(expect.any(Function), 10 * 60 * 1000); expect(cleanupControl.runReplayCleanupTick).toHaveBeenCalledTimes(2); }); + + it("logs the wrapped Postgres cause when a cleanup tick fails", async () => { + const cause = Object.assign(new Error("canceling statement due to lock timeout"), { + code: "55P03", + }); + cleanupControl.runReplayCleanupTick.mockRejectedValueOnce(new Error("Failed query", { cause })); + + await startReplayCleanupScheduler(); + await vi.runOnlyPendingTimersAsync(); + + expect(loggerControl.warn).toHaveBeenCalledWith( + "[Instrumentation] Replay cleanup tick failed", + expect.objectContaining({ + error: "Failed query", + errorName: "Error", + errorCause: "canceling statement due to lock timeout", + errorCauseName: "Error", + errorCauseCode: "55P03", + }) + ); + }); + + it("logs batch and deletion progress when a cleanup tick removes rows", async () => { + cleanupControl.runReplayCleanupTick.mockResolvedValue({ + status: "completed", + batches: 2, + deleted: 150, + }); + + await startReplayCleanupScheduler(); + await vi.runOnlyPendingTimersAsync(); + + expect(loggerControl.info).toHaveBeenCalledWith( + "[Instrumentation] Replay cleanup tick completed", + { status: "completed", batches: 2, deleted: 150 } + ); + }); }); diff --git a/tests/unit/proxy/replay-store.test.ts b/tests/unit/proxy/replay-store.test.ts index 0b2b9dcc3..50e6cdfb3 100644 --- a/tests/unit/proxy/replay-store.test.ts +++ b/tests/unit/proxy/replay-store.test.ts @@ -6,6 +6,7 @@ import { type ReplayMeta, type ReplayPersistedRow, ReplayStore, + resolveReplayCompletedTtlSeconds, resolveReplayTtlSeconds, } from "@/app/v1/_lib/proxy/replay/replay-store"; @@ -21,7 +22,10 @@ import { const envControl = vi.hoisted(() => ({ shouldThrow: false, replayTtlSeconds: 600, - completedTtlSeconds: 3600, +})); + +const runtimeSettingsControl = vi.hoisted(() => ({ + replayCacheTtlMinutes: 30 as number | null, })); const redisControl = vi.hoisted(() => ({ @@ -63,7 +67,6 @@ vi.mock("@/lib/config/env.schema", async (importOriginal) => { return { ...baseEnv, REPLAY_TTL_SECONDS: envControl.replayTtlSeconds, - REPLAY_COMPLETED_TTL_SECONDS: envControl.completedTtlSeconds, }; }, }; @@ -73,6 +76,13 @@ vi.mock("@/lib/redis/client", () => ({ getRedisClient: () => redisControl.client, })); +vi.mock("@/lib/system-settings/proxy-runtime", () => ({ + getCachedProxyRuntimeSettings: () => + runtimeSettingsControl.replayCacheTtlMinutes === null + ? null + : { replayCacheTtlMinutes: runtimeSettingsControl.replayCacheTtlMinutes }, +})); + vi.mock("@/drizzle/db", () => ({ db: { execute: async (query: unknown) => { @@ -259,7 +269,7 @@ function toSqlText(condition: unknown): string { beforeEach(() => { envControl.shouldThrow = false; envControl.replayTtlSeconds = 600; - envControl.completedTtlSeconds = 3600; + runtimeSettingsControl.replayCacheTtlMinutes = 30; redisControl.client = createFakeRedis(); dbState.insertValues = []; dbState.onConflictCalls = 0; @@ -561,8 +571,8 @@ describe("ReplayStore：owner 租约", () => { }); describe("ReplayStore：PG 完成持久层", () => { - it("persistCompleted 写入行（expiresAt = now + REPLAY_COMPLETED_TTL_SECONDS），写路径不顺带清理", async () => { - envControl.completedTtlSeconds = 1000; + it("persistCompleted 按系统设置写入 30 分钟有效期,写路径不顺带清理", async () => { + runtimeSettingsControl.replayCacheTtlMinutes = 30; const store = new ReplayStore(); const row = makePersistedRow(); @@ -587,8 +597,8 @@ describe("ReplayStore：PG 完成持久层", () => { sourceMessageRequestId: 77, }); const expiresAt = (inserted.expiresAt as Date).getTime(); - expect(expiresAt).toBeGreaterThanOrEqual(before + 1000 * 1000); - expect(expiresAt).toBeLessThanOrEqual(after + 1000 * 1000); + expect(expiresAt).toBeGreaterThanOrEqual(before + 30 * 60 * 1000); + expect(expiresAt).toBeLessThanOrEqual(after + 30 * 60 * 1000); expect(dbState.onConflictCalls).toBe(1); // 过期行清理只归定时调度器：写路径不做机会式扫尾 @@ -676,6 +686,9 @@ describe("ReplayStore：PG 完成持久层", () => { expect(deleteSql).toContain("for update skip locked"); expect(deleteSql).toContain("delete from replay_payloads"); expect(deleteSql).toContain("returning 1"); + expect(dialect.sqlToQuery(dbState.executeQueries[0] as SQL).params[0]).toBe( + cutoff.toISOString() + ); }); it("findCompleted 只按 replayId + 未过期条件查询并返回首行", async () => { @@ -699,7 +712,7 @@ describe("ReplayStore：PG 完成持久层", () => { }); }); -describe("resolveReplayTtlSeconds / getReplayStore", () => { +describe("Replay TTL resolver / getReplayStore", () => { it("读 env 的 REPLAY_TTL_SECONDS", () => { envControl.replayTtlSeconds = 1234; expect(resolveReplayTtlSeconds()).toBe(1234); @@ -710,6 +723,21 @@ describe("resolveReplayTtlSeconds / getReplayStore", () => { expect(resolveReplayTtlSeconds()).toBe(600); }); + it("Redis 热层 TTL 不超过系统设置的 Replay 窗口", () => { + runtimeSettingsControl.replayCacheTtlMinutes = 5; + expect(resolveReplayTtlSeconds()).toBe(300); + }); + + it.each([ + { minutes: null, expected: 1800 }, + { minutes: 4, expected: 300 }, + { minutes: 121, expected: 7200 }, + { minutes: 30.9, expected: 1800 }, + ])("PG Replay TTL 规范化 $minutes 分钟为 $expected 秒", ({ minutes, expected }) => { + runtimeSettingsControl.replayCacheTtlMinutes = minutes; + expect(resolveReplayCompletedTtlSeconds()).toBe(expected); + }); + it("getReplayStore 返回共享单例", () => { const first = getReplayStore(); expect(getReplayStore()).toBe(first); diff --git a/tests/unit/proxy/stream-gate-mode-resolution.test.ts b/tests/unit/proxy/stream-gate-mode-resolution.test.ts index 0577fbb40..17464806b 100644 --- a/tests/unit/proxy/stream-gate-mode-resolution.test.ts +++ b/tests/unit/proxy/stream-gate-mode-resolution.test.ts @@ -32,6 +32,9 @@ function createSettings(overrides: Partial = {}): Partial { vi.clearAllMocks(); vi.resetModules(); - getEnvConfigMock.mockReturnValue({ STREAM_GATE_MODE: "off" }); + getEnvConfigMock.mockReturnValue({ + STREAM_GATE_MODE: "off", + ENABLE_REQUEST_REPLAY: false, + ENABLE_CACHE_EFFECTIVENESS: true, + }); }); describe("getProxyRuntimeSettings / getCachedProxyRuntimeSettings", () => { @@ -54,27 +61,46 @@ describe("getProxyRuntimeSettings / getCachedProxyRuntimeSettings", () => { expect(getCachedProxyRuntimeSettings()).toBeNull(); }); - test("getProxyRuntimeSettings 从系统设置缓存映射两字段并更新快照", async () => { + test("getProxyRuntimeSettings 从系统设置缓存映射代理字段并更新快照", async () => { getCachedSystemSettingsMock.mockResolvedValue( createSettings({ streamGateMode: "shadow", affinityIgnoreClientSessionId: false }) ); const { getProxyRuntimeSettings, getCachedProxyRuntimeSettings } = await loadModules(); const settings = await getProxyRuntimeSettings(); - expect(settings).toEqual({ streamGateMode: "shadow", affinityIgnoreClientSessionId: false }); + expect(settings).toEqual({ + streamGateMode: "shadow", + affinityIgnoreClientSessionId: false, + replayEnabled: false, + replayCacheTtlMinutes: 30, + cacheEffectivenessEnabled: true, + }); expect(getCachedProxyRuntimeSettings()).toEqual({ streamGateMode: "shadow", affinityIgnoreClientSessionId: false, + replayEnabled: false, + replayCacheTtlMinutes: 30, + cacheEffectivenessEnabled: true, }); }); test("系统设置读取异常且无快照时回退 env（affinity 默认开）", async () => { getCachedSystemSettingsMock.mockRejectedValue(new Error("db down")); - getEnvConfigMock.mockReturnValue({ STREAM_GATE_MODE: "shadow" }); + getEnvConfigMock.mockReturnValue({ + STREAM_GATE_MODE: "shadow", + ENABLE_REQUEST_REPLAY: false, + ENABLE_CACHE_EFFECTIVENESS: true, + }); const { getProxyRuntimeSettings } = await loadModules(); const settings = await getProxyRuntimeSettings(); - expect(settings).toEqual({ streamGateMode: "shadow", affinityIgnoreClientSessionId: true }); + expect(settings).toEqual({ + streamGateMode: "shadow", + affinityIgnoreClientSessionId: true, + replayEnabled: false, + replayCacheTtlMinutes: 30, + cacheEffectivenessEnabled: true, + }); }); test("系统设置读取异常但已有快照时返回旧快照", async () => { @@ -84,7 +110,13 @@ describe("getProxyRuntimeSettings / getCachedProxyRuntimeSettings", () => { getCachedSystemSettingsMock.mockRejectedValueOnce(new Error("db down")); const settings = await getProxyRuntimeSettings(); - expect(settings).toEqual({ streamGateMode: "off", affinityIgnoreClientSessionId: true }); + expect(settings).toEqual({ + streamGateMode: "off", + affinityIgnoreClientSessionId: true, + replayEnabled: false, + replayCacheTtlMinutes: 30, + cacheEffectivenessEnabled: true, + }); }); }); diff --git a/tests/unit/repository/system-config-degradation-ladder.test.ts b/tests/unit/repository/system-config-degradation-ladder.test.ts index 0996e8b72..c1f120d7c 100644 --- a/tests/unit/repository/system-config-degradation-ladder.test.ts +++ b/tests/unit/repository/system-config-degradation-ladder.test.ts @@ -7,6 +7,7 @@ import type { UpdateSystemSettingsInput } from "@/types/system-config"; // 近代新增列（最新在前），降级链按引入顺序逐层累计剥离。 const RECENT_COLUMNS = [ + "replayCacheTtlMinutes", "cacheEffectivenessEnabled", "replayEnabled", "affinityIgnoreClientSessionId", @@ -27,8 +28,9 @@ const RECENT_COLUMNS = [ "allowNonConversationEndpointProviderFallback", ] as const; -// 全量字段集（46 列）。 +// 全量字段集. const FULL_COLUMNS = [ + "replayCacheTtlMinutes", "cacheEffectivenessEnabled", "replayEnabled", "affinityIgnoreClientSessionId", @@ -189,7 +191,7 @@ describe("SystemSettings：列降级阶梯的尝试序列锁定", () => { const selectMock = vi.fn((selection: Record) => { selections.push(sortedKeys(selection)); callIndex += 1; - if (callIndex < 20) { + if (callIndex < RECENT_COLUMNS.length + 2) { return createRejectingSelectQuery({ code: "42703" }); } return createResolvingSelectQuery([ @@ -222,14 +224,17 @@ describe("SystemSettings：列降级阶梯的尝试序列锁定", () => { const result = await getSystemSettings(); - expect(selectMock).toHaveBeenCalledTimes(20); - // 第 19 次（近代链末层）不含这些新列；第 20 次（passThrough 世代）重新包含旧列。 - expect(selections[18]).not.toContain("enableThinkingEffortConflictRectifier"); - expect(selections[18]).not.toContain("allowNonConversationEndpointProviderFallback"); - expect(selections[18]).toContain("passThroughUpstreamErrorMessage"); - expect(selections[19]).toContain("enableThinkingEffortConflictRectifier"); - expect(selections[19]).toContain("allowNonConversationEndpointProviderFallback"); - expect(selections[19]).not.toContain("passThroughUpstreamErrorMessage"); + const lastRecentIndex = RECENT_COLUMNS.length; + const passThroughIndex = lastRecentIndex + 1; + expect(selectMock).toHaveBeenCalledTimes(passThroughIndex + 1); + expect(selections[lastRecentIndex]).not.toContain("enableThinkingEffortConflictRectifier"); + expect(selections[lastRecentIndex]).not.toContain( + "allowNonConversationEndpointProviderFallback" + ); + expect(selections[lastRecentIndex]).toContain("passThroughUpstreamErrorMessage"); + expect(selections[passThroughIndex]).toContain("enableThinkingEffortConflictRectifier"); + expect(selections[passThroughIndex]).toContain("allowNonConversationEndpointProviderFallback"); + expect(selections[passThroughIndex]).not.toContain("passThroughUpstreamErrorMessage"); // 世代字段集选出的真实值要透传，缺失列由 transformer 落默认值。 expect(result.siteTitle).toBe("Era Row"); @@ -309,7 +314,7 @@ describe("SystemSettings：列降级阶梯的尝试序列锁定", () => { "system_settings 表列缺失，请执行数据库迁移以升级数据库结构。" ); - expect(updateMock).toHaveBeenCalledTimes(22); + expect(updateMock).toHaveBeenCalledTimes(23); const expectedReturningSequence = [ [...FULL_COLUMNS], diff --git a/tests/unit/repository/system-config-update-missing-columns.test.ts b/tests/unit/repository/system-config-update-missing-columns.test.ts index 9b26aed05..b968e07e4 100644 --- a/tests/unit/repository/system-config-update-missing-columns.test.ts +++ b/tests/unit/repository/system-config-update-missing-columns.test.ts @@ -299,10 +299,12 @@ describe("SystemSettings：数据库缺列时的保存兜底", () => { vi.setSystemTime(now); // 第一次 select(fullSelection) 因新列缺失而抛 42703； - // 第二次 select(去掉 cacheEffectivenessEnabled)命中——验证新列已加入降级链最外层。 + // 第二次仅去掉最新的 replayCacheTtlMinutes 后仍失败; + // 第三次累计去掉 cacheEffectivenessEnabled 后命中. const selectMock = vi .fn() .mockReturnValueOnce(createRejectedThenableQuery({ code: "42703" })) + .mockReturnValueOnce(createRejectedThenableQuery({ code: "42703" })) .mockReturnValueOnce( createThenableQuery([ { @@ -335,23 +337,22 @@ describe("SystemSettings：数据库缺列时的保存兜底", () => { const result = await getSystemSettings(); // 降级读取成功（未抛错），缺失列由 transformer 落默认值。 - expect(selectMock).toHaveBeenCalledTimes(2); + expect(selectMock).toHaveBeenCalledTimes(3); expect(result.siteTitle).toBe("CC Hub"); expect(result.enableHttp2).toBe(true); expect(result.affinityIgnoreClientSessionId).toBe(true); expect(result.streamGateMode).toBe("enforce"); - // 关键回归保护：第二次 select 必须恰好剥离了最新列（最外层降级）， - // 而非旧行为先剥离更早引入的列。若新列未加入降级链最外层，下面断言会失败。 - const secondSelection = selectMock.mock.calls[1]?.[0] as Record; - expect(secondSelection).not.toHaveProperty("cacheEffectivenessEnabled"); - expect(secondSelection).toHaveProperty("replayEnabled"); - expect(secondSelection).toHaveProperty("affinityIgnoreClientSessionId"); - expect(secondSelection).toHaveProperty("streamGateMode"); - expect(secondSelection).toHaveProperty("stickyTimeoutCooldownMs"); - expect(secondSelection).toHaveProperty("racingTotalTimeoutMs"); - expect(secondSelection).toHaveProperty("enableGeminiFunctionIdRectifier"); - expect(secondSelection).toHaveProperty("enableThinkingEffortConflictRectifier"); + const thirdSelection = selectMock.mock.calls[2]?.[0] as Record; + expect(thirdSelection).not.toHaveProperty("replayCacheTtlMinutes"); + expect(thirdSelection).not.toHaveProperty("cacheEffectivenessEnabled"); + expect(thirdSelection).toHaveProperty("replayEnabled"); + expect(thirdSelection).toHaveProperty("affinityIgnoreClientSessionId"); + expect(thirdSelection).toHaveProperty("streamGateMode"); + expect(thirdSelection).toHaveProperty("stickyTimeoutCooldownMs"); + expect(thirdSelection).toHaveProperty("racingTotalTimeoutMs"); + expect(thirdSelection).toHaveProperty("enableGeminiFunctionIdRectifier"); + expect(thirdSelection).toHaveProperty("enableThinkingEffortConflictRectifier"); vi.useRealTimers(); }); diff --git a/tests/unit/settings/system-settings-form-replay-cache-toggles.test.tsx b/tests/unit/settings/system-settings-form-replay-cache-toggles.test.tsx index b7f15aa2c..4ebde2a0b 100644 --- a/tests/unit/settings/system-settings-form-replay-cache-toggles.test.tsx +++ b/tests/unit/settings/system-settings-form-replay-cache-toggles.test.tsx @@ -67,6 +67,7 @@ const baseSettings = { ipExtractionConfig: null, // null = 跟随环境变量：本组用例的核心前置 replayEnabled: null, + replayCacheTtlMinutes: 30, cacheEffectivenessEnabled: null, } satisfies Pick< SystemSettings, @@ -100,6 +101,7 @@ const baseSettings = { | "ipGeoLookupEnabled" | "ipExtractionConfig" | "replayEnabled" + | "replayCacheTtlMinutes" | "cacheEffectivenessEnabled" >; @@ -206,4 +208,48 @@ describe("SystemSettingsForm replay/cache-effectiveness null 三态", () => { unmount(); }); + + test("显示默认 Replay 缓存时间并随系统设置提交", async () => { + const { unmount } = render( + + ); + + const ttlInput = document.getElementById("replay-cache-ttl-minutes") as HTMLInputElement | null; + expect(ttlInput?.value).toBe("30"); + + await act(async () => { + if (!ttlInput) throw new Error("未找到 Replay 缓存时间输入框"); + const setter = Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, "value")?.set; + setter?.call(ttlInput, "45"); + ttlInput.dispatchEvent(new Event("input", { bubbles: true })); + ttlInput.dispatchEvent(new Event("change", { bubbles: true })); + await Promise.resolve(); + }); + await submitForm(); + + expect(systemConfigActionMocks.saveSystemSettings).toHaveBeenCalledWith( + expect.objectContaining({ replayCacheTtlMinutes: 45 }) + ); + + unmount(); + }); + + test("Replay 缓存时间错误码显示本地化消息", async () => { + systemConfigActionMocks.saveSystemSettings.mockResolvedValueOnce({ + ok: false, + error: "One or more fields are invalid.", + errorCode: "REPLAY_CACHE_TTL_INVALID", + }); + const { unmount } = render( + + ); + + await submitForm(); + + expect(sonnerMocks.toast.error).toHaveBeenCalledWith( + "Replay cache duration must be a whole number from 5 to 120 minutes." + ); + + unmount(); + }); }); From be9a81937c467eedae532967d8f5723530f89ebb Mon Sep 17 00:00:00 2001 From: Ding <44717411+ding113@users.noreply.github.com> Date: Fri, 7 Aug 2026 17:04:20 +0800 Subject: [PATCH 02/12] fix(keys): preserve last-enabled-key error code (#1393) --- src/actions/keys.ts | 8 ++++---- src/lib/utils/error-messages.ts | 1 + .../actions/keys-self-service-authz.test.ts | 14 +++++++++++++ tests/unit/api/v1/api-client-actions.test.ts | 20 +++++++++++++++++++ 4 files changed, 39 insertions(+), 4 deletions(-) diff --git a/src/actions/keys.ts b/src/actions/keys.ts index 1b9241b30..23dd7ee52 100644 --- a/src/actions/keys.ts +++ b/src/actions/keys.ts @@ -505,7 +505,7 @@ export async function editKey( return { ok: false, error: tError("CANNOT_DISABLE_LAST_KEY"), - errorCode: ERROR_CODES.OPERATION_FAILED, + errorCode: ERROR_CODES.CANNOT_DISABLE_LAST_KEY, }; } } @@ -1263,7 +1263,7 @@ export async function toggleKeyEnabled(keyId: number, enabled: boolean): Promise return { ok: false, error: tError("CANNOT_DISABLE_LAST_KEY"), - errorCode: ERROR_CODES.OPERATION_FAILED, + errorCode: ERROR_CODES.CANNOT_DISABLE_LAST_KEY, }; } } @@ -1418,7 +1418,7 @@ export async function batchUpdateKeys( if (currentEnabledCount - disableCount < 1) { throw new BatchUpdateError( tError("CANNOT_DISABLE_LAST_KEY"), - ERROR_CODES.OPERATION_FAILED + ERROR_CODES.CANNOT_DISABLE_LAST_KEY ); } } @@ -1478,7 +1478,7 @@ export async function batchUpdateKeys( if (Number(remainingEnabled?.count ?? 0) < 1) { throw new BatchUpdateError( tError("CANNOT_DISABLE_LAST_KEY"), - ERROR_CODES.OPERATION_FAILED + ERROR_CODES.CANNOT_DISABLE_LAST_KEY ); } } diff --git a/src/lib/utils/error-messages.ts b/src/lib/utils/error-messages.ts index eb71f53d5..3c59e283d 100644 --- a/src/lib/utils/error-messages.ts +++ b/src/lib/utils/error-messages.ts @@ -101,6 +101,7 @@ export const BUSINESS_ERRORS = { USER_LIMITS_RESET_PARTIAL_FAILURE: "USER_LIMITS_RESET_PARTIAL_FAILURE", USER_STATS_RESET_PARTIAL_FAILURE: "USER_STATS_RESET_PARTIAL_FAILURE", CANNOT_DELETE_LAST_KEY: "CANNOT_DELETE_LAST_KEY", + CANNOT_DISABLE_LAST_KEY: "CANNOT_DISABLE_LAST_KEY", CANNOT_DELETE_LAST_GROUP_KEY: "CANNOT_DELETE_LAST_GROUP_KEY", KEY_NOT_FOUND: "KEY_NOT_FOUND", } as const; diff --git a/tests/unit/actions/keys-self-service-authz.test.ts b/tests/unit/actions/keys-self-service-authz.test.ts index 236a4bf95..391d1c210 100644 --- a/tests/unit/actions/keys-self-service-authz.test.ts +++ b/tests/unit/actions/keys-self-service-authz.test.ts @@ -193,6 +193,20 @@ describe("toggleKeyEnabled self-service authorization", () => { expect(result.ok).toBe(true); expect(updateKeyMock).toHaveBeenCalledWith(42, { is_enabled: false }); }); + + it("returns the dedicated business code when disabling the last enabled key", async () => { + getSessionMock.mockResolvedValue(webSession); + countActiveKeysByUserMock.mockResolvedValue(1); + + const { toggleKeyEnabled } = await import("@/actions/keys"); + const result = await toggleKeyEnabled(42, false); + + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.errorCode).toBe("CANNOT_DISABLE_LAST_KEY"); + } + expect(updateKeyMock).not.toHaveBeenCalled(); + }); }); describe("renewKeyExpiresAt self-service authorization", () => { diff --git a/tests/unit/api/v1/api-client-actions.test.ts b/tests/unit/api/v1/api-client-actions.test.ts index 36307abee..e8884a418 100644 --- a/tests/unit/api/v1/api-client-actions.test.ts +++ b/tests/unit/api/v1/api-client-actions.test.ts @@ -598,6 +598,26 @@ describe("v1 action compatibility client", () => { }); }); + test("preserves the last-enabled-key business code through toggleKeyEnabled", async () => { + postMock.mockRejectedValueOnce( + new ApiError({ + status: 400, + errorCode: "CANNOT_DISABLE_LAST_KEY", + detail: "Bad request", + }) + ); + + const result = await keys.toggleKeyEnabled(7, false); + + expect(postMock).toHaveBeenCalledWith("/api/v1/keys/7:enable", { enabled: false }, undefined); + expect(result).toEqual({ + ok: false, + error: "Bad request", + errorCode: "CANNOT_DISABLE_LAST_KEY", + errorParams: undefined, + }); + }); + test("maps key.action_failed through toVoidActionResult to OPERATION_FAILED", async () => { deleteMock.mockRejectedValueOnce( new ApiError({ status: 400, errorCode: "key.action_failed", detail: "Bad request" }) From 08e866069a6f04180f6d66c03e67c96b67640969 Mon Sep 17 00:00:00 2001 From: ding113 Date: Fri, 7 Aug 2026 19:45:34 +0800 Subject: [PATCH 03/12] fix(replay): enforce cache TTL boundary validation and error mapping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The toSystemSettings transformer now validates that replayCacheTtlMinutes is an integer within the min/max bounds (5–120), falling back to the default for any out-of-range or non-integer value. Previously, arbitrary values from the database would pass through unchecked. The API update handler now maps replay TTL validation failures to the REPLAY_CACHE_TTL_INVALID error code via getReplayCacheTtlValidationErrorCode, so clients receive a precise error instead of a generic validation failure. Additional review-driven fixes: - Schema default references the shared REPLAY_CACHE_TTL_MINUTES_DEFAULT constant instead of a magic number - Import paths in schemas.ts normalized to absolute module specifiers - Tests cover boundary values (5, 120), upper-limit rejection (121), and invalid-value fallback in the transformer - Degradation ladder test includes replayCacheTtlMinutes in its update payload and returning-column assertions - Form test uses the i18n message key instead of a hardcoded string --- src/app/api/v1/resources/system/handlers.ts | 5 ++++- src/drizzle/schema.ts | 5 ++++- src/lib/validation/schemas.ts | 6 +++--- src/repository/_shared/transformers.test.ts | 11 +++++++++++ src/repository/_shared/transformers.ts | 16 ++++++++++++++-- tests/api/v1/system/system-config.test.ts | 19 +++++++++++++++++++ tests/unit/actions/system-config-save.test.ts | 9 +++++++++ .../system-config-degradation-ladder.test.ts | 5 +++-- ...ettings-form-replay-cache-toggles.test.tsx | 2 +- 9 files changed, 68 insertions(+), 10 deletions(-) diff --git a/src/app/api/v1/resources/system/handlers.ts b/src/app/api/v1/resources/system/handlers.ts index 21df4a018..f238060b8 100644 --- a/src/app/api/v1/resources/system/handlers.ts +++ b/src/app/api/v1/resources/system/handlers.ts @@ -9,6 +9,7 @@ import { parseHonoJsonBody } from "@/lib/api/v1/_shared/request-body"; import { jsonResponse } from "@/lib/api/v1/_shared/response-helpers"; import { SystemSettingsUpdateSchema } from "@/lib/api/v1/schemas/system-config"; import { getDiscoveryValidationErrorCode } from "@/lib/validation/discovery-settings"; +import { getReplayCacheTtlValidationErrorCode } from "@/lib/validation/replay-settings"; export async function getSystemSettings(c: Context): Promise { const actions = await import("@/actions/system-config"); @@ -29,7 +30,9 @@ export async function getSystemDisplaySettings(_c: Context): Promise { export async function updateSystemSettings(c: Context): Promise { const body = await parseHonoJsonBody(c, SystemSettingsUpdateSchema, { - validationErrorCode: (error) => getDiscoveryValidationErrorCode(error.issues), + validationErrorCode: (error) => + getDiscoveryValidationErrorCode(error.issues) ?? + getReplayCacheTtlValidationErrorCode(error.issues), }); if (!body.ok) return body.response; const actions = await import("@/actions/system-config"); diff --git a/src/drizzle/schema.ts b/src/drizzle/schema.ts index 60f497c0a..fea464b10 100644 --- a/src/drizzle/schema.ts +++ b/src/drizzle/schema.ts @@ -22,6 +22,7 @@ import type { FilterOperation } from "@/lib/request-filter-types"; import type { IpExtractionConfig } from "@/types/ip-extraction"; import type { AuditCategory } from "@/types/audit-log"; import type { RoutingTraceV1 } from "@/types/routing-trace"; +import { REPLAY_CACHE_TTL_MINUTES_DEFAULT } from "@/lib/validation/replay-settings"; // Enums export const dailyResetModeEnum = pgEnum('daily_reset_mode', ['fixed', 'rolling']); @@ -1057,7 +1058,9 @@ export const systemSettings = pgTable('system_settings', { // F2 Replay 开关覆写（null = 跟随环境变量 ENABLE_REQUEST_REPLAY） replayEnabled: boolean('replay_enabled'), // F2 Replay 完成 payload 的可重放窗口(分钟,默认 30) - replayCacheTtlMinutes: integer('replay_cache_ttl_minutes').notNull().default(30), + replayCacheTtlMinutes: integer('replay_cache_ttl_minutes') + .notNull() + .default(REPLAY_CACHE_TTL_MINUTES_DEFAULT), // F3b 最长前缀匹配缓存模拟开关覆写（null = 跟随环境变量 ENABLE_CACHE_EFFECTIVENESS） cacheEffectivenessEnabled: boolean('cache_effectiveness_enabled'), diff --git a/src/lib/validation/schemas.ts b/src/lib/validation/schemas.ts index 58c259857..bf9540f61 100644 --- a/src/lib/validation/schemas.ts +++ b/src/lib/validation/schemas.ts @@ -20,17 +20,17 @@ import { DISCOVERY_FIELD_LIMITS, DISCOVERY_SETTINGS_INVALID_ERROR_CODE, DISCOVERY_WINDOW_INVALID_ERROR_CODE, -} from "./discovery-settings"; +} from "@/lib/validation/discovery-settings"; import { REPLAY_CACHE_TTL_INVALID_ERROR_CODE, REPLAY_CACHE_TTL_MINUTES_MAX, REPLAY_CACHE_TTL_MINUTES_MIN, -} from "./replay-settings"; +} from "@/lib/validation/replay-settings"; export { DISCOVERY_SETTINGS_INVALID_ERROR_CODE, DISCOVERY_WINDOW_INVALID_ERROR_CODE, -} from "./discovery-settings"; +} from "@/lib/validation/discovery-settings"; const CACHE_TTL_PREFERENCE = z.enum(["inherit", "5m", "1h"]); const CONTEXT_1M_PREFERENCE = z.enum(["inherit", "force_enable", "disabled"]); diff --git a/src/repository/_shared/transformers.test.ts b/src/repository/_shared/transformers.test.ts index b9c515058..460917b8e 100644 --- a/src/repository/_shared/transformers.test.ts +++ b/src/repository/_shared/transformers.test.ts @@ -300,6 +300,17 @@ describe("src/repository/_shared/transformers.ts", () => { expect(toSystemSettings({ replayCacheTtlMinutes: 45 }).replayCacheTtlMinutes).toBe(45); }); + it.each([5, 120])("应保留 Replay 缓存时间的有效边界 %s", (value) => { + expect(toSystemSettings({ replayCacheTtlMinutes: value }).replayCacheTtlMinutes).toBe(value); + }); + + it.each([0, 4, 121, -1, 5.5, Number.NaN, "45"])( + "应将无效 Replay 缓存时间 %s 回退为默认值", + (value) => { + expect(toSystemSettings({ replayCacheTtlMinutes: value }).replayCacheTtlMinutes).toBe(30); + } + ); + it("应映射 interceptAnthropicWarmupRequests 字段", () => { const result = toSystemSettings({ id: 1, diff --git a/src/repository/_shared/transformers.ts b/src/repository/_shared/transformers.ts index f401d6485..b49454fd8 100644 --- a/src/repository/_shared/transformers.ts +++ b/src/repository/_shared/transformers.ts @@ -2,7 +2,11 @@ import { PROVIDER_TIMEOUT_DEFAULTS } from "@/lib/constants/provider.constants"; import { normalizeProviderModelRedirectRules } from "@/lib/provider-model-redirects"; import { DEFAULT_SITE_TITLE } from "@/lib/site-title"; import { formatCostForStorage } from "@/lib/utils/currency"; -import { REPLAY_CACHE_TTL_MINUTES_DEFAULT } from "@/lib/validation/replay-settings"; +import { + REPLAY_CACHE_TTL_MINUTES_DEFAULT, + REPLAY_CACHE_TTL_MINUTES_MAX, + REPLAY_CACHE_TTL_MINUTES_MIN, +} from "@/lib/validation/replay-settings"; import type { Key } from "@/types/key"; import type { MessageRequest } from "@/types/message"; import type { ModelPrice } from "@/types/model-price"; @@ -245,6 +249,14 @@ export function toSystemSettings(dbSettings: any): SystemSettings { maxJsonDepth: 200, maxFixSize: 1024 * 1024, }; + const replayCacheTtlMinutes = dbSettings?.replayCacheTtlMinutes; + const normalizedReplayCacheTtlMinutes = + typeof replayCacheTtlMinutes === "number" && + Number.isInteger(replayCacheTtlMinutes) && + replayCacheTtlMinutes >= REPLAY_CACHE_TTL_MINUTES_MIN && + replayCacheTtlMinutes <= REPLAY_CACHE_TTL_MINUTES_MAX + ? replayCacheTtlMinutes + : REPLAY_CACHE_TTL_MINUTES_DEFAULT; return { id: dbSettings?.id ?? 0, @@ -321,7 +333,7 @@ export function toSystemSettings(dbSettings: any): SystemSettings { : "enforce", affinityIgnoreClientSessionId: dbSettings?.affinityIgnoreClientSessionId ?? true, replayEnabled: dbSettings?.replayEnabled ?? null, - replayCacheTtlMinutes: dbSettings?.replayCacheTtlMinutes ?? REPLAY_CACHE_TTL_MINUTES_DEFAULT, + replayCacheTtlMinutes: normalizedReplayCacheTtlMinutes, cacheEffectivenessEnabled: dbSettings?.cacheEffectivenessEnabled ?? null, createdAt: dbSettings?.createdAt ? new Date(dbSettings.createdAt) : new Date(), updatedAt: dbSettings?.updatedAt ? new Date(dbSettings.updatedAt) : new Date(), diff --git a/tests/api/v1/system/system-config.test.ts b/tests/api/v1/system/system-config.test.ts index c16d0eac6..440c23239 100644 --- a/tests/api/v1/system/system-config.test.ts +++ b/tests/api/v1/system/system-config.test.ts @@ -189,6 +189,25 @@ describe("v1 system config endpoints", () => { }); expect(invalidReplayTtl.response.status).toBe(400); expect(invalidReplayTtl.json).toMatchObject({ errorCode: "REPLAY_CACHE_TTL_INVALID" }); + + const invalidReplayTtlUpper = await callV1Route({ + method: "PUT", + pathname: "/api/v1/system/settings", + headers: { Authorization: "Bearer admin-token" }, + body: { replayCacheTtlMinutes: 121 }, + }); + expect(invalidReplayTtlUpper.response.status).toBe(400); + expect(invalidReplayTtlUpper.json).toMatchObject({ errorCode: "REPLAY_CACHE_TTL_INVALID" }); + + for (const value of [5, 120]) { + const validReplayTtl = await callV1Route({ + method: "PUT", + pathname: "/api/v1/system/settings", + headers: { Authorization: "Bearer admin-token" }, + body: { replayCacheTtlMinutes: value }, + }); + expect(validReplayTtl.response.status).toBe(200); + } }); test("returns a stable error code for out-of-range Discovery settings", async () => { diff --git a/tests/unit/actions/system-config-save.test.ts b/tests/unit/actions/system-config-save.test.ts index ebc5cc160..df00e73db 100644 --- a/tests/unit/actions/system-config-save.test.ts +++ b/tests/unit/actions/system-config-save.test.ts @@ -161,6 +161,15 @@ describe("saveSystemSettings", () => { ); }); + it.each([5, 120])("accepts Replay cache TTL boundary %s", async (value) => { + const result = await saveSystemSettings({ replayCacheTtlMinutes: value }); + + expect(result.ok).toBe(true); + expect(updateSystemSettingsMock).toHaveBeenCalledWith( + expect.objectContaining({ replayCacheTtlMinutes: value }) + ); + }); + it.each([4, 121, 30.5])("rejects invalid Replay cache TTL %s", async (value) => { const result = await saveSystemSettings({ replayCacheTtlMinutes: value }); diff --git a/tests/unit/repository/system-config-degradation-ladder.test.ts b/tests/unit/repository/system-config-degradation-ladder.test.ts index c1f120d7c..5469eaf3c 100644 --- a/tests/unit/repository/system-config-degradation-ladder.test.ts +++ b/tests/unit/repository/system-config-degradation-ladder.test.ts @@ -292,6 +292,7 @@ describe("SystemSettings：列降级阶梯的尝试序列锁定", () => { const payload: UpdateSystemSettingsInput = { siteTitle: "Ladder Pin", + replayCacheTtlMinutes: 45, codexPriorityBillingSource: "actual", billNonSuccessfulRequests: true, billHedgeLosers: false, @@ -314,8 +315,6 @@ describe("SystemSettings：列降级阶梯的尝试序列锁定", () => { "system_settings 表列缺失，请执行数据库迁移以升级数据库结构。" ); - expect(updateMock).toHaveBeenCalledTimes(23); - const expectedReturningSequence = [ [...FULL_COLUMNS], ...RECENT_COLUMNS.map((_, index) => omit(FULL_COLUMNS, RECENT_COLUMNS.slice(0, index + 1))), @@ -323,11 +322,13 @@ describe("SystemSettings：列降级阶梯的尝试序列锁定", () => { omit(FULL_COLUMNS, HIGH_CONCURRENCY_ERA_OMIT), omit(FULL_COLUMNS, CODEX_ERA_RETURNING_OMIT), ].map(sorted); + expect(updateMock).toHaveBeenCalledTimes(expectedReturningSequence.length); expect(returningKeySequence).toEqual(expectedReturningSequence); const fullSetKeys = [ "updatedAt", "siteTitle", + "replayCacheTtlMinutes", "codexPriorityBillingSource", "billNonSuccessfulRequests", "billHedgeLosers", diff --git a/tests/unit/settings/system-settings-form-replay-cache-toggles.test.tsx b/tests/unit/settings/system-settings-form-replay-cache-toggles.test.tsx index 4ebde2a0b..696159fdf 100644 --- a/tests/unit/settings/system-settings-form-replay-cache-toggles.test.tsx +++ b/tests/unit/settings/system-settings-form-replay-cache-toggles.test.tsx @@ -247,7 +247,7 @@ describe("SystemSettingsForm replay/cache-effectiveness null 三态", () => { await submitForm(); expect(sonnerMocks.toast.error).toHaveBeenCalledWith( - "Replay cache duration must be a whole number from 5 to 120 minutes." + loadMessages("en").settings.config.form.replayCacheTtlInvalid ); unmount(); From 63ff4cf338c1df94f6a5064e4b7dbfe019a62ab6 Mon Sep 17 00:00:00 2001 From: ding113 Date: Fri, 7 Aug 2026 19:45:34 +0800 Subject: [PATCH 04/12] fix(instrumentation): correct replay cleanup logger argument order The replay cleanup scheduler passed the message string as the first argument and the structured data object as the second, which is the reverse of the pino calling convention. Swap them so the data object is first and the message string is second, matching the rest of the codebase. Test expectations updated accordingly. --- src/instrumentation.ts | 23 +++++++++++-------- .../instrumentation-replay-cleanup.test.ts | 8 +++---- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/src/instrumentation.ts b/src/instrumentation.ts index 45b0b23b6..a14f0003d 100644 --- a/src/instrumentation.ts +++ b/src/instrumentation.ts @@ -331,13 +331,14 @@ export async function startReplayCleanupScheduler(): Promise { void runReplayCleanupTick() .then((result) => { if (result.deleted > 0) { - logger.info("[Instrumentation] Replay cleanup tick completed", result); + logger.info(result, "[Instrumentation] Replay cleanup tick completed"); } }) .catch((error) => { - logger.warn("[Instrumentation] Replay cleanup tick failed", { - ...describeSchedulerError(error), - }); + logger.warn( + { ...describeSchedulerError(error) }, + "[Instrumentation] Replay cleanup tick failed" + ); }); }; @@ -345,13 +346,15 @@ export async function startReplayCleanupScheduler(): Promise { instrumentationState.__CCH_REPLAY_CLEANUP_INTERVAL_ID__ = setInterval(runTick, intervalMs); instrumentationState.__CCH_REPLAY_CLEANUP_STARTED__ = true; - logger.info("[Instrumentation] Replay cleanup scheduler started", { - intervalSeconds: intervalMs / 1000, - }); + logger.info( + { intervalSeconds: intervalMs / 1000 }, + "[Instrumentation] Replay cleanup scheduler started" + ); } catch (error) { - logger.warn("[Instrumentation] Replay cleanup scheduler init failed", { - ...describeSchedulerError(error), - }); + logger.warn( + { ...describeSchedulerError(error) }, + "[Instrumentation] Replay cleanup scheduler init failed" + ); } } diff --git a/tests/unit/instrumentation-replay-cleanup.test.ts b/tests/unit/instrumentation-replay-cleanup.test.ts index 937aca87a..ca6d960c6 100644 --- a/tests/unit/instrumentation-replay-cleanup.test.ts +++ b/tests/unit/instrumentation-replay-cleanup.test.ts @@ -73,14 +73,14 @@ describe("startReplayCleanupScheduler", () => { await vi.runOnlyPendingTimersAsync(); expect(loggerControl.warn).toHaveBeenCalledWith( - "[Instrumentation] Replay cleanup tick failed", expect.objectContaining({ error: "Failed query", errorName: "Error", errorCause: "canceling statement due to lock timeout", errorCauseName: "Error", errorCauseCode: "55P03", - }) + }), + "[Instrumentation] Replay cleanup tick failed" ); }); @@ -95,8 +95,8 @@ describe("startReplayCleanupScheduler", () => { await vi.runOnlyPendingTimersAsync(); expect(loggerControl.info).toHaveBeenCalledWith( - "[Instrumentation] Replay cleanup tick completed", - { status: "completed", batches: 2, deleted: 150 } + { status: "completed", batches: 2, deleted: 150 }, + "[Instrumentation] Replay cleanup tick completed" ); }); }); From 191bb19afdd76c4f49b42791fcf602cc9c0c5602 Mon Sep 17 00:00:00 2001 From: Ding <44717411+ding113@users.noreply.github.com> Date: Sat, 8 Aug 2026 07:37:55 +0800 Subject: [PATCH 05/12] fix(proxy): support remote compaction v2 passthrough (#1404) * fix(proxy): support remote compaction v2 passthrough * fix(proxy): address remote compaction review feedback * fix(proxy): localize normalized request errors * test(proxy): cover localized normalization errors --- messages/en/errors.json | 1 + messages/ja/errors.json | 1 + messages/ru/errors.json | 1 + messages/zh-CN/errors.json | 1 + messages/zh-TW/errors.json | 1 + src/app/v1/_lib/proxy-handler.ts | 3 +- src/app/v1/_lib/proxy/message-service.test.ts | 13 ++ src/app/v1/_lib/proxy/message-service.ts | 4 +- src/app/v1/_lib/proxy/remote-compaction.ts | 24 +++ src/app/v1/_lib/proxy/response-handler.ts | 2 +- .../v1/_lib/proxy/response-input-rectifier.ts | 1 + src/app/v1/_lib/proxy/session.ts | 46 +++++- .../proxy/stream-gate/frame-classifier.ts | 6 + src/lib/utils/error-messages.ts | 1 + .../unit/i18n/session-request-errors.test.ts | 1 + ...rwarder-raw-passthrough-regression.test.ts | 81 +++++++++ .../proxy-handler-public-success.test.ts | 86 +++++++++- tests/unit/proxy/remote-compaction-v2.test.ts | 156 ++++++++++++++++++ .../response-handler-bill-non-success.test.ts | 2 + .../proxy/response-input-rectifier.test.ts | 14 +- .../proxy/stream-gate-content-gate.test.ts | 20 +++ .../stream-gate-frame-classifier.test.ts | 34 ++++ 22 files changed, 485 insertions(+), 14 deletions(-) create mode 100644 src/app/v1/_lib/proxy/remote-compaction.ts create mode 100644 tests/unit/proxy/remote-compaction-v2.test.ts diff --git a/messages/en/errors.json b/messages/en/errors.json index 47202ff9a..68c50b1ac 100644 --- a/messages/en/errors.json +++ b/messages/en/errors.json @@ -38,6 +38,7 @@ "INTERNAL_ERROR": "Internal server error, please try again later", "DATABASE_ERROR": "Database error", + "INVALID_NORMALIZED_BODY": "The normalized request body could not be serialized", "NOT_FOUND": "Resource not found", "OPERATION_FAILED": "Operation failed", "USER_5H_FIXED_RESET_REQUIRES_REDIS": "Resetting a fixed 5H limit requires Redis to be available", diff --git a/messages/ja/errors.json b/messages/ja/errors.json index d6bc74d16..3cce45eb8 100644 --- a/messages/ja/errors.json +++ b/messages/ja/errors.json @@ -38,6 +38,7 @@ "INTERNAL_ERROR": "内部サーバーエラー、後でもう一度お試しください", "DATABASE_ERROR": "データベースエラー", + "INVALID_NORMALIZED_BODY": "正規化されたリクエスト本文をシリアル化できません", "NOT_FOUND": "リソースが見つかりません", "OPERATION_FAILED": "操作に失敗しました", "USER_5H_FIXED_RESET_REQUIRES_REDIS": "固定 5H 制限のリセットには Redis が必要です", diff --git a/messages/ru/errors.json b/messages/ru/errors.json index 29b2f58ae..749e0086f 100644 --- a/messages/ru/errors.json +++ b/messages/ru/errors.json @@ -38,6 +38,7 @@ "INTERNAL_ERROR": "Внутренняя ошибка сервера, попробуйте позже", "DATABASE_ERROR": "Ошибка базы данных", + "INVALID_NORMALIZED_BODY": "Нормализованное тело запроса невозможно сериализовать", "NOT_FOUND": "Ресурс не найден", "OPERATION_FAILED": "Операция не удалась", "USER_5H_FIXED_RESET_REQUIRES_REDIS": "Для сброса фиксированного лимита 5H требуется доступный Redis", diff --git a/messages/zh-CN/errors.json b/messages/zh-CN/errors.json index 1ed2e29cc..b4f12d840 100644 --- a/messages/zh-CN/errors.json +++ b/messages/zh-CN/errors.json @@ -38,6 +38,7 @@ "INTERNAL_ERROR": "系统内部错误，请稍后重试", "DATABASE_ERROR": "数据库错误", + "INVALID_NORMALIZED_BODY": "规范化后的请求体无法序列化", "NOT_FOUND": "资源不存在", "OPERATION_FAILED": "操作失败", "USER_5H_FIXED_RESET_REQUIRES_REDIS": "重置固定 5H 限额需要 Redis 可用", diff --git a/messages/zh-TW/errors.json b/messages/zh-TW/errors.json index ba8dbe595..7a5862914 100644 --- a/messages/zh-TW/errors.json +++ b/messages/zh-TW/errors.json @@ -38,6 +38,7 @@ "INTERNAL_ERROR": "系統內部錯誤，請稍後重試", "DATABASE_ERROR": "資料庫錯誤", + "INVALID_NORMALIZED_BODY": "規範化後的請求體無法序列化", "NOT_FOUND": "資源不存在", "OPERATION_FAILED": "操作失敗", "USER_5H_FIXED_RESET_REQUIRES_REDIS": "重設固定 5H 限額需要 Redis 可用", diff --git a/src/app/v1/_lib/proxy-handler.ts b/src/app/v1/_lib/proxy-handler.ts index fa33a4f59..7c802fec6 100644 --- a/src/app/v1/_lib/proxy-handler.ts +++ b/src/app/v1/_lib/proxy-handler.ts @@ -1,4 +1,5 @@ import type { Context } from "hono"; +import { isRawPassthroughEndpointPolicy } from "@/app/v1/_lib/proxy/endpoint-policy"; import { findSafeDatabaseError } from "@/drizzle/admitted-client"; import { getCachedSystemSettings } from "@/lib/config"; import { logger } from "@/lib/logger"; @@ -169,7 +170,7 @@ export async function handleProxyRequest(c: Context): Promise { // Reuse the system settings already loaded above (with its fallback path) // instead of re-reading the cache. A transient cache miss must not turn an // otherwise-routable request into an error response. - if (cachedSystemSettings) { + if (cachedSystemSettings && !isRawPassthroughEndpointPolicy(session.getEndpointPolicy())) { const fakeStreamingResponse = await tryFakeStreamingPath(session, cachedSystemSettings); if (fakeStreamingResponse) { return await attachSessionIdToErrorResponse(session.sessionId, fakeStreamingResponse); diff --git a/src/app/v1/_lib/proxy/message-service.test.ts b/src/app/v1/_lib/proxy/message-service.test.ts index 1ed77d4b0..d46e26cb0 100644 --- a/src/app/v1/_lib/proxy/message-service.test.ts +++ b/src/app/v1/_lib/proxy/message-service.test.ts @@ -29,6 +29,7 @@ function createSession(providerType: string, message: Record) { userAgent: "codex_cli_rs/1.0.0", clientIp: "127.0.0.1", getEndpoint: () => "/v1/responses", + getManagedEndpoint: () => "/v1/responses", getOriginalModel: () => "gpt-5", setOriginalModel: vi.fn(), getSpecialSettings: () => (specialSettings.length > 0 ? specialSettings : null), @@ -126,4 +127,16 @@ describe("ProxyMessageService Codex reasoning effort audit", () => { }) ); }); + + test("remote compaction v2 uses the compact management endpoint", async () => { + const { session } = createSession("codex", { input: [{ type: "compaction_trigger" }] }); + (session as unknown as { getManagedEndpoint: () => string }).getManagedEndpoint = () => + "/v1/responses/compact"; + + await ProxyMessageService.ensureContext(session); + + expect(createMessageRequestMock).toHaveBeenCalledWith( + expect.objectContaining({ endpoint: "/v1/responses/compact" }) + ); + }); }); diff --git a/src/app/v1/_lib/proxy/message-service.ts b/src/app/v1/_lib/proxy/message-service.ts index d5ea02c57..b891f902a 100644 --- a/src/app/v1/_lib/proxy/message-service.ts +++ b/src/app/v1/_lib/proxy/message-service.ts @@ -26,8 +26,8 @@ export class ProxyMessageService { return; } - // Extract endpoint from URL pathname (nullable) - const endpoint = session.getEndpoint() ?? undefined; + // v2 compaction 记录为 compact 管理端点，以复用非对话日志和计费语义。 + const endpoint = session.getManagedEndpoint?.() ?? session.getEndpoint() ?? undefined; const sessionIdentity = session.getSessionIdentityMetadata(); // 修复模型重定向记录问题： diff --git a/src/app/v1/_lib/proxy/remote-compaction.ts b/src/app/v1/_lib/proxy/remote-compaction.ts new file mode 100644 index 000000000..6aedf53f0 --- /dev/null +++ b/src/app/v1/_lib/proxy/remote-compaction.ts @@ -0,0 +1,24 @@ +import { normalizeEndpointPath, V1_ENDPOINT_PATHS } from "@/app/v1/_lib/proxy/endpoint-paths"; + +/** + * 识别 Responses input 中精确的 Remote Compaction v2 trigger。 + * 单对象 input 与数组 input 使用同一 item 语义，其他类型不会被推断为 compaction。 + */ +export function isRemoteCompactionV2Request(pathname: string, requestBody: unknown): boolean { + if (normalizeEndpointPath(pathname) !== V1_ENDPOINT_PATHS.RESPONSES) { + return false; + } + if (typeof requestBody !== "object" || requestBody === null) { + return false; + } + + const input = (requestBody as Record).input; + const items = Array.isArray(input) ? input : [input]; + + return items.some( + (item) => + typeof item === "object" && + item !== null && + (item as Record).type === "compaction_trigger" + ); +} diff --git a/src/app/v1/_lib/proxy/response-handler.ts b/src/app/v1/_lib/proxy/response-handler.ts index efd564ceb..81e78c2cf 100644 --- a/src/app/v1/_lib/proxy/response-handler.ts +++ b/src/app/v1/_lib/proxy/response-handler.ts @@ -1266,7 +1266,7 @@ function buildCostCalculationOptions( } function isNonBillingUsageEndpoint(session: ProxySession): boolean { - return isNonBillingEndpoint(session.getEndpoint()); + return isNonBillingEndpoint(session.getManagedEndpoint()); } function hasBillableInputCostPerRequest(priceData: { input_cost_per_request?: unknown }): boolean { diff --git a/src/app/v1/_lib/proxy/response-input-rectifier.ts b/src/app/v1/_lib/proxy/response-input-rectifier.ts index a301fe442..55839aff6 100644 --- a/src/app/v1/_lib/proxy/response-input-rectifier.ts +++ b/src/app/v1/_lib/proxy/response-input-rectifier.ts @@ -89,6 +89,7 @@ export async function normalizeResponseInput(session: ProxySession): Promise { @@ -1193,6 +1197,33 @@ export class ProxySession { return this.endpointPolicy; } + /** + * 在请求 message 被原地规范化后，同步 raw wire body 与审计日志。 + * 标准数组请求不会调用此方法，因此原始请求字节仍保持不变。 + */ + async syncRequestBodyFromMessage(): Promise { + const serialized = JSON.stringify(this.request.message); + if (serialized === undefined) { + const { getLocale } = await import("next-intl/server"); + const message = await getErrorMessageServer( + await getLocale(), + ERROR_CODES.INVALID_NORMALIZED_BODY + ); + throw new ProxyError(message, 400); + } + + this.request.buffer = new TextEncoder().encode(serialized).buffer; + this.request.log = JSON.stringify(optimizeRequestMessage(this.request.message), null, 2); + } + + /** + * 获取管理语义的 endpoint。 + * Remote Compaction v2 保留真实 /v1/responses wire path，但复用 v1 compact 的策略、日志和计费分类。 + */ + getManagedEndpoint(): string { + return this.managedEndpoint ?? this.getEndpoint() ?? "/"; + } + /** * 获取请求的 API endpoint（来自 URL.pathname） * 处理边界：若 URL 不存在则返回 null @@ -1554,15 +1585,20 @@ function optimizeRequestMessage(message: Record): Record +): string { try { const pathname = requestUrl.pathname; if (typeof pathname === "string" && pathname.length > 0) { - return resolveEndpointPolicy(pathname); + return isRemoteCompactionV2Request(pathname, requestMessage) + ? V1_ENDPOINT_PATHS.RESPONSES_COMPACT + : pathname; } } catch {} - return resolveEndpointPolicy("/"); + return "/"; } export function extractModelFromPath(pathname: string): string | null { diff --git a/src/app/v1/_lib/proxy/stream-gate/frame-classifier.ts b/src/app/v1/_lib/proxy/stream-gate/frame-classifier.ts index f89995484..b6dac5c55 100644 --- a/src/app/v1/_lib/proxy/stream-gate/frame-classifier.ts +++ b/src/app/v1/_lib/proxy/stream-gate/frame-classifier.ts @@ -167,6 +167,12 @@ const STREAM_SIGNALS: Record = { eventTypes: ["response.code_interpreter_call_code.done"], anyPaths: ["code"], }, + { + // Remote/server-side compaction 的 opaque state 是完整协议 payload，只有非空且类型精确匹配才提交。 + eventTypes: ["response.output_item.done"], + anyPaths: ["item.encrypted_content"], + valueMatches: [{ path: "item.type", values: ["compaction"] }], + }, { // output_item.added 的 name/id/status 只是结构元数据；真实 payload 到达前不能提交， // 否则紧随其后的 response.failed / 断流将失去透明 fallback 机会。 diff --git a/src/lib/utils/error-messages.ts b/src/lib/utils/error-messages.ts index 3c59e283d..d2aadff5b 100644 --- a/src/lib/utils/error-messages.ts +++ b/src/lib/utils/error-messages.ts @@ -74,6 +74,7 @@ export const AUTH_ERRORS = { export const SERVER_ERRORS = { INTERNAL_ERROR: "INTERNAL_ERROR", DATABASE_ERROR: "DATABASE_ERROR", + INVALID_NORMALIZED_BODY: "INVALID_NORMALIZED_BODY", NOT_FOUND: "NOT_FOUND", OPERATION_FAILED: "OPERATION_FAILED", CREATE_FAILED: "CREATE_FAILED", diff --git a/tests/unit/i18n/session-request-errors.test.ts b/tests/unit/i18n/session-request-errors.test.ts index 0d75aaf64..8450ba3ea 100644 --- a/tests/unit/i18n/session-request-errors.test.ts +++ b/tests/unit/i18n/session-request-errors.test.ts @@ -4,6 +4,7 @@ import { describe, expect, test } from "vitest"; const LOCALES = ["zh-CN", "zh-TW", "en", "ja", "ru"] as const; const ERROR_CODES = [ + "INVALID_NORMALIZED_BODY", "SESSION_REQUEST_SOURCE_MISMATCH", "SESSION_REQUEST_SELECTOR_INCOMPLETE", ] as const; diff --git a/tests/unit/proxy/proxy-forwarder-raw-passthrough-regression.test.ts b/tests/unit/proxy/proxy-forwarder-raw-passthrough-regression.test.ts index cf331202f..67495901f 100644 --- a/tests/unit/proxy/proxy-forwarder-raw-passthrough-regression.test.ts +++ b/tests/unit/proxy/proxy-forwarder-raw-passthrough-regression.test.ts @@ -29,6 +29,7 @@ vi.mock("@/lib/proxy-agent", () => ({ import { resolveEndpointPolicy } from "@/app/v1/_lib/proxy/endpoint-policy"; import { ProxyForwarder } from "@/app/v1/_lib/proxy/forwarder"; +import { rectifyResponseInput } from "@/app/v1/_lib/proxy/response-input-rectifier"; import { ProxySession } from "@/app/v1/_lib/proxy/session"; import type { Provider } from "@/types/provider"; @@ -149,6 +150,86 @@ describe("ProxyForwarder raw passthrough regression", () => { expect(readBodyText(capturedInit?.body)).toBe(originalBody); }); + it("remote compaction v2 保留 /v1/responses wire path 与原始请求体", async () => { + const originalBody = + '{\n "model": "gpt-5.5",\n "stream": true,\n "input": [{"type":"compaction_trigger"}]\n}\n'; + const upstreamSse = + 'event: response.output_item.done\ndata: {"type":"response.output_item.done","output_index":0,"item":{"type":"compaction","encrypted_content":"opaque-state"}}\n\n' + + 'event: response.completed\ndata: {"type":"response.completed","response":{"id":"resp_compact","status":"completed","output":[{"type":"compaction","encrypted_content":"opaque-state"}]}}\n\n'; + const session = createRawPassthroughSession(originalBody, { + "x-codex-beta-features": "remote_compaction_v2", + }); + session.requestUrl = new URL("https://proxy.example.com/v1/responses?transport=http"); + const provider = createProvider(); + + let capturedUrl: string | null = null; + let capturedBody: BodyInit | undefined; + let capturedHeaders: Headers | null = null; + const fetchWithoutAutoDecode = vi.spyOn(ProxyForwarder as any, "fetchWithoutAutoDecode"); + fetchWithoutAutoDecode.mockImplementationOnce(async (url: string, init: RequestInit) => { + capturedUrl = url; + capturedBody = init.body ?? undefined; + capturedHeaders = new Headers(init.headers); + return new Response(upstreamSse, { + status: 200, + headers: { "content-type": "text/event-stream" }, + }); + }); + + const { doForward } = ProxyForwarder as unknown as { + doForward: (session: ProxySession, provider: Provider, baseUrl: string) => Promise; + }; + + const response = await doForward(session, provider, provider.url); + + expect(new URL(capturedUrl as string).pathname).toBe("/v1/responses"); + expect(new URL(capturedUrl as string).searchParams.get("transport")).toBe("http"); + expect(readBodyText(capturedBody)).toBe(originalBody); + expect(capturedHeaders?.get("x-codex-beta-features")).toBe("remote_compaction_v2"); + expect(await response.text()).toBe(upstreamSse); + }); + + it("remote compaction v2 将单对象 input 规范化后再透传", async () => { + const originalBody = '{"model":"gpt-5.5","stream":true,"input":{"type":"compaction_trigger"}}'; + const session = createRawPassthroughSession(originalBody, { + "x-codex-beta-features": "remote_compaction_v2", + }); + session.requestUrl = new URL("https://proxy.example.com/v1/responses?transport=http"); + const provider = createProvider(); + + const result = rectifyResponseInput(session.request.message); + expect(result.applied).toBe(true); + await session.syncRequestBodyFromMessage(); + + let capturedUrl: string | null = null; + let capturedBody: BodyInit | undefined; + let capturedHeaders: Headers | null = null; + const fetchWithoutAutoDecode = vi.spyOn(ProxyForwarder as any, "fetchWithoutAutoDecode"); + fetchWithoutAutoDecode.mockImplementationOnce(async (url: string, init: RequestInit) => { + capturedUrl = url; + capturedBody = init.body ?? undefined; + capturedHeaders = new Headers(init.headers); + return new Response("{}", { + status: 200, + headers: { "content-type": "application/json", "content-length": "2" }, + }); + }); + + const { doForward } = ProxyForwarder as unknown as { + doForward: (session: ProxySession, provider: Provider, baseUrl: string) => Promise; + }; + + await doForward(session, provider, provider.url); + + expect(new URL(capturedUrl as string).pathname).toBe("/v1/responses"); + expect(new URL(capturedUrl as string).searchParams.get("transport")).toBe("http"); + expect(JSON.parse(readBodyText(capturedBody) ?? "{}").input).toEqual([ + { type: "compaction_trigger" }, + ]); + expect(capturedHeaders?.get("content-length")).toBeNull(); + expect(capturedHeaders?.get("x-codex-beta-features")).toBe("remote_compaction_v2"); + }); + it("raw passthrough 出站请求不得继续携带 transfer-encoding 这类 hop-by-hop 头", async () => { const body = '{"model":"gpt-5.5","input":[]}'; const session = createRawPassthroughSession(body, { diff --git a/tests/unit/proxy/proxy-handler-public-success.test.ts b/tests/unit/proxy/proxy-handler-public-success.test.ts index 99a9a0675..0b15e2c86 100644 --- a/tests/unit/proxy/proxy-handler-public-success.test.ts +++ b/tests/unit/proxy/proxy-handler-public-success.test.ts @@ -17,8 +17,11 @@ const boundary = vi.hoisted(() => ({ loadSettings: vi.fn<() => Promise>(), runGuards: vi.fn<(session: ProxySession) => Promise>(), send: vi.fn<(session: ProxySession) => Promise>(), + fakeStreamingCalls: 0, })); +let observedSession: ProxySession | null = null; + vi.mock("@/lib/config", async (importOriginal) => ({ ...(await importOriginal()), getCachedSystemSettings: boundary.loadSettings, @@ -39,6 +42,21 @@ vi.mock("@/app/v1/_lib/proxy/forwarder", () => ({ ProxyForwarder: { send: boundary.send }, })); +vi.mock("@/app/v1/_lib/proxy/fake-streaming/proxy-integration", async (importOriginal) => { + const actual = + await importOriginal(); + + return { + ...actual, + tryFakeStreamingPath: async ( + ...args: Parameters + ): Promise => { + boundary.fakeStreamingCalls += 1; + return await actual.tryFakeStreamingPath(...args); + }, + }; +}); + vi.mock("@/lib/session-tracker", () => ({ SessionTracker: { decrementConcurrentCount: boundary.decrementConcurrentCount, @@ -84,13 +102,18 @@ function createContext(pathname: string, body: Record): Context describe("handleProxyRequest public success behavior", () => { beforeEach(() => { + observedSession = null; + boundary.fakeStreamingCalls = 0; boundary.runGuards.mockReset(); boundary.send.mockReset(); boundary.incrementConcurrentCount.mockReset(); boundary.decrementConcurrentCount.mockReset(); boundary.loadSettings.mockReset(); boundary.loadSettings.mockResolvedValue(defaultSettings); - boundary.runGuards.mockResolvedValue(null); + boundary.runGuards.mockImplementation(async (session) => { + observedSession = session; + return null; + }); boundary.incrementConcurrentCount.mockResolvedValue(undefined); boundary.decrementConcurrentCount.mockResolvedValue(undefined); }); @@ -144,6 +167,7 @@ describe("handleProxyRequest public success behavior", () => { expect(body).toContain('"text":"generated"'); expect(body).toContain("event: message_stop"); expect(boundary.send).toHaveBeenCalledOnce(); + expect(boundary.fakeStreamingCalls).toBe(1); }); it("normalizes Responses input and output at the public boundary", async () => { @@ -174,4 +198,64 @@ describe("handleProxyRequest public success behavior", () => { tools: [], }); }); + + it("routes remote compaction v2 through the v1 compact management policy", async () => { + boundary.send.mockResolvedValue( + new Response( + JSON.stringify({ + output: [{ type: "compaction", encrypted_content: "opaque-state" }], + }), + { status: 200, headers: { "content-type": "application/json" } } + ) + ); + + const response = await handleProxyRequest( + createContext("/v1/responses", { + model: "gpt-5-codex", + input: [{ role: "user", content: "keep" }, { type: "compaction_trigger" }], + }) + ); + + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ + output: [{ type: "compaction", encrypted_content: "opaque-state" }], + }); + expect(observedSession?.getEndpoint()).toBe("/v1/responses"); + expect(observedSession?.getManagedEndpoint()).toBe("/v1/responses/compact"); + expect(observedSession?.getEndpointPolicy().kind).toBe("raw_passthrough"); + expect(boundary.fakeStreamingCalls).toBe(0); + }); + + it("normalizes object-form remote compaction before raw passthrough", async () => { + boundary.loadSettings.mockResolvedValue({ + ...defaultSettings, + fakeStreamingWhitelist: [{ model: "gpt-5-codex", groupTags: [] }], + }); + boundary.send.mockResolvedValue( + new Response( + JSON.stringify({ + output: [{ type: "compaction", encrypted_content: "opaque-state" }], + }), + { status: 200, headers: { "content-type": "application/json" } } + ) + ); + + const response = await handleProxyRequest( + createContext("/v1/responses", { + model: "gpt-5-codex", + stream: true, + input: { type: "compaction_trigger" }, + }) + ); + + expect(response.status).toBe(200); + expect(observedSession?.getManagedEndpoint()).toBe("/v1/responses/compact"); + expect(observedSession?.getEndpointPolicy().kind).toBe("raw_passthrough"); + expect(observedSession?.request.message.input).toEqual([{ type: "compaction_trigger" }]); + expect(JSON.parse(new TextDecoder().decode(observedSession?.request.buffer)).input).toEqual([ + { type: "compaction_trigger" }, + ]); + expect(boundary.fakeStreamingCalls).toBe(0); + expect(boundary.send).toHaveBeenCalledOnce(); + }); }); diff --git a/tests/unit/proxy/remote-compaction-v2.test.ts b/tests/unit/proxy/remote-compaction-v2.test.ts new file mode 100644 index 000000000..8ed590009 --- /dev/null +++ b/tests/unit/proxy/remote-compaction-v2.test.ts @@ -0,0 +1,156 @@ +import type { Context } from "hono"; +import { describe, expect, it, vi } from "vitest"; + +vi.mock("@/repository/model-price", () => ({ + findLatestPriceByModel: vi.fn(), +})); + +vi.mock("@/repository/system-config", () => ({ + getSystemSettings: vi.fn(), +})); + +vi.mock("@/lib/config/system-settings-cache", () => ({ + getCachedSystemSettings: vi.fn(async () => ({ enableResponseInputRectifier: true })), +})); + +vi.mock("next-intl/server", () => ({ + getLocale: vi.fn(async () => "en"), + getTranslations: vi.fn(async () => (key: string) => { + if (key === "INVALID_NORMALIZED_BODY") { + return "The normalized request body could not be serialized"; + } + return key; + }), +})); + +import { V1_ENDPOINT_PATHS } from "@/app/v1/_lib/proxy/endpoint-paths"; +import { resolveEndpointPolicy } from "@/app/v1/_lib/proxy/endpoint-policy"; +import { isRemoteCompactionV2Request } from "@/app/v1/_lib/proxy/remote-compaction"; +import { normalizeResponseInput } from "@/app/v1/_lib/proxy/response-input-rectifier"; +import { ProxySession } from "@/app/v1/_lib/proxy/session"; +import { isNonBillingEndpoint } from "@/lib/utils/performance-formatter"; + +function makeContext(url: string, body: string): Context { + const request = new Request(url, { + method: "POST", + headers: { "content-type": "application/json" }, + body, + }); + + return { + req: { + method: "POST", + url, + raw: request, + header: (name?: string) => { + if (name === undefined) { + return Object.fromEntries(request.headers.entries()); + } + return request.headers.get(name) ?? undefined; + }, + }, + } as unknown as Context; +} + +describe("remote compaction v2 request classification", () => { + it("recognizes an exact compaction_trigger item on the Responses endpoint", () => { + expect( + isRemoteCompactionV2Request(V1_ENDPOINT_PATHS.RESPONSES, { + input: [{ role: "user", content: "keep this" }, { type: "compaction_trigger" }], + }) + ).toBe(true); + }); + + it("recognizes a single compaction_trigger input object", () => { + expect( + isRemoteCompactionV2Request(V1_ENDPOINT_PATHS.RESPONSES, { + input: { type: "compaction_trigger" }, + }) + ).toBe(true); + }); + + it.each([ + ["different endpoint", V1_ENDPOINT_PATHS.CHAT_COMPLETIONS, [{ type: "compaction_trigger" }]], + ["future item type", V1_ENDPOINT_PATHS.RESPONSES, [{ type: "compaction_trigger_v2" }]], + ["nested marker", V1_ENDPOINT_PATHS.RESPONSES, [{ content: { type: "compaction_trigger" } }]], + ["string marker", V1_ENDPOINT_PATHS.RESPONSES, ["compaction_trigger"]], + ["unrelated input object", V1_ENDPOINT_PATHS.RESPONSES, { type: "message" }], + [ + "compaction replay item", + V1_ENDPOINT_PATHS.RESPONSES, + [{ type: "compaction", encrypted_content: "opaque-state" }], + ], + ])("does not infer compaction from %s", (_label, pathname, input) => { + expect(isRemoteCompactionV2Request(pathname, { input })).toBe(false); + }); + + it("reuses v1 compact management while preserving the v2 wire path and body", async () => { + const body = + '{\n "model": "gpt-5-codex",\n "stream": true,\n "input": [{"role":"user","content":"keep"},{"type":"compaction_trigger"}]\n}\n'; + const session = await ProxySession.fromContext( + makeContext("https://hub.test/v1/responses", body) + ); + + expect(session.getEndpoint()).toBe(V1_ENDPOINT_PATHS.RESPONSES); + expect(session.getManagedEndpoint()).toBe(V1_ENDPOINT_PATHS.RESPONSES_COMPACT); + expect(session.getEndpointPolicy()).toBe( + resolveEndpointPolicy(V1_ENDPOINT_PATHS.RESPONSES_COMPACT) + ); + expect(isNonBillingEndpoint(session.getManagedEndpoint())).toBe(true); + expect(new TextDecoder().decode(session.request.buffer)).toBe(body); + }); + + it("keeps normal Responses requests on the conversation policy", async () => { + const body = JSON.stringify({ + model: "gpt-5-codex", + stream: true, + input: [{ type: "message", role: "user", content: "hello" }], + }); + const session = await ProxySession.fromContext( + makeContext("https://hub.test/v1/responses", body) + ); + + expect(session.getManagedEndpoint()).toBe(V1_ENDPOINT_PATHS.RESPONSES); + expect(session.getEndpointPolicy().kind).toBe("default"); + }); + + it("normalizes a single compaction trigger across message, buffer, and log", async () => { + const session = await ProxySession.fromContext( + makeContext( + "https://hub.test/v1/responses", + JSON.stringify({ + model: "gpt-5-codex", + stream: true, + input: { type: "compaction_trigger" }, + }) + ) + ); + + await normalizeResponseInput(session); + + expect(session.getManagedEndpoint()).toBe(V1_ENDPOINT_PATHS.RESPONSES_COMPACT); + expect(session.getEndpointPolicy().kind).toBe("raw_passthrough"); + expect(session.request.message.input).toEqual([{ type: "compaction_trigger" }]); + expect(JSON.parse(new TextDecoder().decode(session.request.buffer))).toMatchObject({ + input: [{ type: "compaction_trigger" }], + }); + expect(JSON.parse(session.request.log)).toMatchObject({ + input: [{ type: "compaction_trigger" }], + }); + }); + + it("returns a localized 400 error when the normalized body cannot be serialized", async () => { + const session = await ProxySession.fromContext( + makeContext( + "https://hub.test/v1/responses", + JSON.stringify({ model: "gpt-5-codex", input: [] }) + ) + ); + session.request.message = undefined as unknown as Record; + + await expect(session.syncRequestBodyFromMessage()).rejects.toMatchObject({ + message: "The normalized request body could not be serialized", + statusCode: 400, + }); + }); +}); diff --git a/tests/unit/proxy/response-handler-bill-non-success.test.ts b/tests/unit/proxy/response-handler-bill-non-success.test.ts index 3b9ae2674..2ab9acda2 100644 --- a/tests/unit/proxy/response-handler-bill-non-success.test.ts +++ b/tests/unit/proxy/response-handler-bill-non-success.test.ts @@ -54,6 +54,7 @@ import { detectUpstreamErrorFromSseOrJsonText } from "@/lib/utils/upstream-error const mockGetCachedSystemSettings = getCachedSystemSettings as unknown as ReturnType; type MinimalSession = { + getManagedEndpoint: () => string; getEndpoint: () => string | null; getOriginalModel: () => string | null; getCurrentModel: () => string | null; @@ -62,6 +63,7 @@ type MinimalSession = { function makeSession(): MinimalSession { return { + getManagedEndpoint: () => "/v1/messages", getEndpoint: () => "/v1/messages", getOriginalModel: () => "claude-3-5-sonnet", getCurrentModel: () => "claude-3-5-sonnet", diff --git a/tests/unit/proxy/response-input-rectifier.test.ts b/tests/unit/proxy/response-input-rectifier.test.ts index a77922d96..9982a7655 100644 --- a/tests/unit/proxy/response-input-rectifier.test.ts +++ b/tests/unit/proxy/response-input-rectifier.test.ts @@ -19,14 +19,17 @@ const getCachedMock = vi.mocked(getCachedSystemSettings); function createMockSession(input: unknown): { session: ProxySession; specialSettings: SpecialSetting[]; + syncRequestBodyFromMessage: ReturnType; } { const specialSettings: SpecialSetting[] = []; + const syncRequestBodyFromMessage = vi.fn(); const session = { request: { message: { model: "gpt-4o", input } }, sessionId: "sess_test", addSpecialSetting: (s: SpecialSetting) => specialSettings.push(s), + syncRequestBodyFromMessage, } as unknown as ProxySession; - return { session, specialSettings }; + return { session, specialSettings, syncRequestBodyFromMessage }; } describe("rectifyResponseInput", () => { @@ -184,7 +187,7 @@ describe("normalizeResponseInput", () => { it("normalizes string input and records audit when enabled", async () => { getCachedMock.mockResolvedValue({ enableResponseInputRectifier: true } as any); - const { session, specialSettings } = createMockSession("hello"); + const { session, specialSettings, syncRequestBodyFromMessage } = createMockSession("hello"); await normalizeResponseInput(session); const message = session.request.message as Record; @@ -198,29 +201,32 @@ describe("normalizeResponseInput", () => { action: "string_to_array", originalType: "string", }); + expect(syncRequestBodyFromMessage).toHaveBeenCalledOnce(); }); it("skips normalization when feature is disabled", async () => { getCachedMock.mockResolvedValue({ enableResponseInputRectifier: false } as any); - const { session, specialSettings } = createMockSession("hello"); + const { session, specialSettings, syncRequestBodyFromMessage } = createMockSession("hello"); await normalizeResponseInput(session); const message = session.request.message as Record; expect(message.input).toBe("hello"); expect(specialSettings).toHaveLength(0); + expect(syncRequestBodyFromMessage).not.toHaveBeenCalled(); }); it("does not record audit for passthrough (array input)", async () => { getCachedMock.mockResolvedValue({ enableResponseInputRectifier: true } as any); const arrayInput = [{ role: "user", content: [{ type: "input_text", text: "hi" }] }]; - const { session, specialSettings } = createMockSession(arrayInput); + const { session, specialSettings, syncRequestBodyFromMessage } = createMockSession(arrayInput); await normalizeResponseInput(session); const message = session.request.message as Record; expect(message.input).toBe(arrayInput); expect(specialSettings).toHaveLength(0); + expect(syncRequestBodyFromMessage).not.toHaveBeenCalled(); }); it("wraps single object input and records audit when enabled", async () => { diff --git a/tests/unit/proxy/stream-gate-content-gate.test.ts b/tests/unit/proxy/stream-gate-content-gate.test.ts index f0bd9324e..bd97a540a 100644 --- a/tests/unit/proxy/stream-gate-content-gate.test.ts +++ b/tests/unit/proxy/stream-gate-content-gate.test.ts @@ -234,6 +234,26 @@ describe("runStreamContentGate", () => { expect(result.readerDone).toBe(false); }); + it("openai-responses: commits a compaction item before response.completed", async () => { + const compaction = + 'event: response.output_item.done\ndata: {"type":"response.output_item.done","item":{"type":"compaction","encrypted_content":"opaque-state"}}\n\n'; + const completed = + 'event: response.completed\ndata: {"type":"response.completed","response":{"status":"completed"}}\n\n'; + const reader = readerFromChunks([compaction, completed]); + + const result = await runStreamContentGate(reader, { + ...GATE_OPTIONS, + family: "openai-responses", + }); + + expect(result.committed).toBe(true); + if (!result.committed) return; + expect(await drainPrefix(result.prefixChunks)).toBe(compaction); + expect(result.readerDone).toBe(false); + const rest = await reader.read(); + expect(new TextDecoder().decode(rest.value)).toBe(completed); + }); + it("gemini: usage-only chunks buffer until content commits", async () => { const reader = readerFromChunks([ 'data: {"usageMetadata":{"totalTokenCount":1}}\n\n', diff --git a/tests/unit/proxy/stream-gate-frame-classifier.test.ts b/tests/unit/proxy/stream-gate-frame-classifier.test.ts index 0a139bbeb..3e8fb3053 100644 --- a/tests/unit/proxy/stream-gate-frame-classifier.test.ts +++ b/tests/unit/proxy/stream-gate-frame-classifier.test.ts @@ -290,6 +290,40 @@ describe("classifyFrame: openai-responses", () => { ).toBe("content"); }); + it("content: compaction output item with opaque encrypted content", () => { + expect( + classifyFrame( + "openai-responses", + "response.output_item.done", + '{"type":"response.output_item.done","item":{"type":"compaction","encrypted_content":"opaque-state"}}' + ) + ).toBe("content"); + }); + + it("neutral: empty or non-compaction encrypted output item", () => { + expect( + classifyFrame( + "openai-responses", + "response.output_item.done", + '{"type":"response.output_item.done","item":{"type":"compaction","encrypted_content":""}}' + ) + ).toBe("neutral"); + expect( + classifyFrame( + "openai-responses", + "response.output_item.done", + '{"type":"response.output_item.done","item":{"type":"reasoning","encrypted_content":"opaque-state"}}' + ) + ).toBe("neutral"); + expect( + classifyFrame( + "openai-responses", + "response.output_item.added", + '{"type":"response.output_item.added","item":{"type":"compaction","encrypted_content":"opaque-state"}}' + ) + ).toBe("neutral"); + }); + it("neutral: output_item.added without item.name (message item)", () => { expect( classifyFrame( From 1ac2b8fa57f69c84ebe6bd2ca616ab8625bd0ea2 Mon Sep 17 00:00:00 2001 From: Ding <44717411+ding113@users.noreply.github.com> Date: Mon, 10 Aug 2026 16:06:55 +0800 Subject: [PATCH 06/12] fix(proxy): prevent shared mutation and replay memory retention (#1405) * fix(proxy): prevent shared mutation and replay memory retention Rectifiers and cache TTL overrides mutated shared nested arrays in place, leaking attempt-specific edits across concurrent shadow sessions and the original request. Switch to copy-on-write: filtered arrays are assigned through the top-level message object, TTL override rebuilds only changed message entries, and shadow sessions shallow-copy the request message while sharing the readonly buffer instead of deep-cloning multi-MB request bodies per shadow. ReplaySpool retained full payload while Redis or PG writes blocked and abort/disable raced with in-flight flushes. Payload is now snapshotted and cleared before persistence; abort immediately releases accumulated parts and queued batches, deduplicates concurrent calls through a shared barrier, and fences store cleanup through the writeChain so concurrency quota is freed only after cleanup completes. Streaming detection now reads the stream flag directly from the outgoing message instead of re-parsing the serialized body. * fix(types): resolve tsgo type inference failures for zod 4 body parsing tsgo (TypeScript native preview) could not infer the generic output type from JsonBodySchema's structural safeParse signature because zod 4 uses this-type polymorphism (core.output) instead of a plain generic. This left body.data as unknown across all v1 API handlers, producing 36 TS18046/TS2698/TS2345 errors. Switch parseHonoJsonBody/parseJsonBody/parseJson to infer the schema type directly () and extract the output via z.output, which tsgo resolves correctly. Add explicit parameter annotations to four zod .transform()/.refine() callbacks where tsgo also failed to infer the this-dependent input type. CI Run: https://github.com/ding113/claude-code-hub/actions/runs/31245382874 Co-Authored-By: Claude Opus 4.6 --------- Co-authored-by: claude[bot] <41898282+claude[bot]@users.noreply.github.com> Co-authored-by: Claude Opus 4.6 --- .../api/v1/resources/providers/handlers.ts | 10 +- .../v1/_lib/proxy/billing-header-rectifier.ts | 8 +- src/app/v1/_lib/proxy/forwarder.ts | 30 +-- src/app/v1/_lib/proxy/replay/replay-spool.ts | 88 +++++--- src/lib/api/v1/_shared/request-body.ts | 16 +- src/lib/api/v1/schemas/audit-logs.ts | 4 +- src/lib/api/v1/schemas/me.ts | 2 +- src/lib/api/v1/schemas/system-config.ts | 2 +- src/lib/api/v1/schemas/usage-logs.ts | 2 +- .../proxy/billing-header-rectifier.test.ts | 15 +- tests/unit/proxy/cache-ttl-override.test.ts | 29 +-- .../proxy-forwarder-hedge-first-byte.test.ts | 28 +++ tests/unit/proxy/replay-spool.test.ts | 202 +++++++++++++++++- 13 files changed, 351 insertions(+), 85 deletions(-) diff --git a/src/app/api/v1/resources/providers/handlers.ts b/src/app/api/v1/resources/providers/handlers.ts index 93398344c..3129765f5 100644 --- a/src/app/api/v1/resources/providers/handlers.ts +++ b/src/app/api/v1/resources/providers/handlers.ts @@ -1,5 +1,4 @@ import type { Context } from "hono"; -import type { ZodError } from "zod"; import { z } from "zod"; import type { ActionResult } from "@/actions/types"; import { hasLegacyRedactedWritePlaceholders } from "@/lib/api/legacy-action-sanitizers"; @@ -773,11 +772,10 @@ function providerNotFound(c: Context): Response { }); } -type JsonBodySchema = { - safeParse: (value: unknown) => { success: true; data: T } | { success: false; error: ZodError }; -}; - -async function parseJson(c: Context, schema: JsonBodySchema): Promise { +async function parseJson( + c: Context, + schema: S +): Promise | Response> { const body = await parseHonoJsonBody(c, schema); if (!body.ok) return body.response; return body.data; diff --git a/src/app/v1/_lib/proxy/billing-header-rectifier.ts b/src/app/v1/_lib/proxy/billing-header-rectifier.ts index f52693dc5..e12a97270 100644 --- a/src/app/v1/_lib/proxy/billing-header-rectifier.ts +++ b/src/app/v1/_lib/proxy/billing-header-rectifier.ts @@ -20,7 +20,7 @@ const BILLING_HEADER_PATTERN = /^\s*x-anthropic-billing-header\s*:/i; /** * Remove x-anthropic-billing-header text blocks from the request system prompt. - * Mutates the message object in place (matches existing rectifier conventions). + * Writes changes back through the top-level message object without mutating shared nested arrays. */ export function rectifyBillingHeader( message: Record @@ -62,11 +62,7 @@ export function rectifyBillingHeader( } if (extractedValues.length > 0) { - // Mutate in place: replace system array contents - system.length = 0; - for (const item of filtered) { - system.push(item); - } + message.system = filtered; return { applied: true, removedCount: extractedValues.length, extractedValues }; } diff --git a/src/app/v1/_lib/proxy/forwarder.ts b/src/app/v1/_lib/proxy/forwarder.ts index a0e8a8701..84bc26942 100644 --- a/src/app/v1/_lib/proxy/forwarder.ts +++ b/src/app/v1/_lib/proxy/forwarder.ts @@ -543,17 +543,23 @@ export function applyCacheTtlOverrideToMessage( // messages[].content[] const messages = message.messages; if (Array.isArray(messages)) { - for (const msg of messages) { + let nextMessages: unknown[] | null = null; + for (let index = 0; index < messages.length; index += 1) { + const msg = messages[index]; if (!msg || typeof msg !== "object") continue; const msgObj = msg as Record; const content = msgObj.content; if (!Array.isArray(content)) continue; const result = applyTtlToContentBlocks(content, ttl); if (result.applied) { - msgObj.content = result.blocks; + nextMessages ??= [...messages]; + nextMessages[index] = { ...msgObj, content: result.blocks }; applied = true; } } + if (nextMessages) { + message.messages = nextMessages; + } } return applied; @@ -1166,7 +1172,11 @@ async function tryApplyReactiveRectifier(params: { } const requestDetailsBeforeRectify = buildRequestDetails(requestSession); - const rectified = descriptor.rectify(requestSession.request.message as Record); + const mutableMessage = structuredClone( + requestSession.request.message as Record + ); + requestSession.request.message = mutableMessage; + const rectified = descriptor.rectify(mutableMessage); addSpecialSettingForPersistence( requestSession, @@ -3325,13 +3335,7 @@ export class ProxyForwarder { const bodyString = JSON.stringify(messageToSend); requestBody = bodyString; session.forwardedRequestBody = bodyString; - - try { - const parsed = JSON.parse(bodyString); - isStreaming = parsed.stream === true; - } catch { - isStreaming = false; - } + isStreaming = messageToSend.stream === true; if (process.env.NODE_ENV === "development") { logger.trace("ProxyForwarder: Forwarding request", { @@ -7770,8 +7774,10 @@ export class ProxyForwarder { shadowState.request = { ...session.request, - message: structuredClone(session.request.message), - buffer: session.request.buffer ? session.request.buffer.slice(0) : undefined, + // attempt 改写采用顶层 copy-on-write；发送前的私有参数过滤会生成独立深拷贝。 + message: { ...session.request.message }, + // 原始请求字节只读；shadow 共享底层 buffer，任何改写都必须整体替换属性。 + buffer: session.request.buffer, imageRequestMetadata: cloneOpenAIImageRequestMetadata(session.request.imageRequestMetadata), }; shadow.requestUrl = new URL(session.requestUrl.toString()); diff --git a/src/app/v1/_lib/proxy/replay/replay-spool.ts b/src/app/v1/_lib/proxy/replay/replay-spool.ts index 3691494c3..11817432e 100644 --- a/src/app/v1/_lib/proxy/replay/replay-spool.ts +++ b/src/app/v1/_lib/proxy/replay/replay-spool.ts @@ -40,6 +40,7 @@ export class ReplaySpool { private readonly store = getReplayStore(); private readonly decoder = new TextDecoder("utf-8"); private readonly parts: string[] = []; + private readonly queuedBatches = new Set(); private pending: string[] = []; private pendingBytes = 0; private totalBytes = 0; @@ -120,11 +121,12 @@ export class ReplaySpool { if (batch.length === 0) return; this.pending = []; this.pendingBytes = 0; + this.queuedBatches.add(batch); // 续接体自带 try/catch：链永不 rejected；每个 await 之后复查 disabled， // 防止与 disable/halt 竞态时在清理之后又写回 owning meta this.writeChain = this.writeChain.then(async () => { try { - if (this.disabled) return; + if (this.disabled || this.aborting) return; const expectedChunkCount = this.chunkCount + batch.length; const appended = await this.store.writeOwned( this.identity.replayId, @@ -132,7 +134,7 @@ export class ReplaySpool { this.buildMeta("owning", { chunkCount: expectedChunkCount }), batch ); - if (this.disabled) return; + if (this.disabled || this.aborting) return; if (appended === null) { // Redis 不可用：本次 replay 放弃（热层写是原子的，不会留下半批数据） this.disable("redis_unavailable"); @@ -145,10 +147,13 @@ export class ReplaySpool { this.chunkCount = appended; this.metaWritten = true; } catch (error) { + if (this.aborting) return; logger.debug("[ReplaySpool] flush failed, disabling spool", { error: error instanceof Error ? error.message : String(error), }); this.disable("flush_error"); + } finally { + this.queuedBatches.delete(batch); } }); } @@ -172,15 +177,15 @@ export class ReplaySpool { private startOwnerHeartbeat(): void { this.ownerHeartbeatTimer = setInterval(() => { - if (this.disabled || this.released || this.ownerHeartbeatInFlight) return; + if (this.disabled || this.aborting || this.released || this.ownerHeartbeatInFlight) return; this.ownerHeartbeatInFlight = true; void this.store .renewOwnerLease(this.identity.replayId, this.ownerToken) .then((leaseHeld) => { - if (!leaseHeld && !this.released) this.halt("owner_lease_lost"); + if (!leaseHeld && !this.aborting && !this.released) this.halt("owner_lease_lost"); }) .catch(() => { - if (!this.released) this.halt("owner_lease_lost"); + if (!this.aborting && !this.released) this.halt("owner_lease_lost"); }) .finally(() => { this.ownerHeartbeatInFlight = false; @@ -193,12 +198,13 @@ export class ReplaySpool { bootstrap(): void { this.writeChain = this.writeChain.then(async () => { try { - if (this.disabled || this.metaWritten) return; + if (this.disabled || this.aborting || this.metaWritten) return; const chunkCount = await this.store.writeOwned( this.identity.replayId, this.ownerToken, this.buildMeta("owning") ); + if (this.disabled || this.aborting) return; if (chunkCount === null) { this.disable("redis_unavailable"); return; @@ -210,6 +216,7 @@ export class ReplaySpool { this.chunkCount = chunkCount; this.metaWritten = true; } catch (error) { + if (this.aborting) return; logger.debug("[ReplaySpool] bootstrap failed, disabling spool", { error: error instanceof Error ? error.message : String(error), }); @@ -234,11 +241,12 @@ export class ReplaySpool { const batch = this.pending; this.pending = []; this.pendingBytes = 0; + this.queuedBatches.add(batch); this.writeChain = this.writeChain.then(async () => { let pgPersisted = false; try { - if (this.disabled) return; + if (this.disabled || this.aborting) return; const expectedChunkCount = this.chunkCount + batch.length; const appended = await this.store.writeOwned( this.identity.replayId, @@ -255,6 +263,7 @@ export class ReplaySpool { } this.chunkCount = appended; this.metaWritten = true; + const payload = this.takePayload(); // 先写 PG（持久 payload），再翻 Redis meta 为 completed（热层可服务） const persistResult = await this.store.persistCompleted({ replayId: this.identity.replayId, @@ -266,7 +275,7 @@ export class ReplaySpool { model: this.identity.model, statusCode: this.statusCode, headers: this.headers, - payload: this.parts.join(""), + payload, byteSize: this.totalBytes, sourceMessageRequestId: messageRequestId, }); @@ -312,6 +321,8 @@ export class ReplaySpool { ) .catch(() => false); } finally { + this.queuedBatches.delete(batch); + this.clearPayload(); this.release(); } }); @@ -320,12 +331,19 @@ export class ReplaySpool { /** 终态失败：meta 置 aborted + 删块；已 aborted 的条目绝不被重放命中。 */ async abort(reason: string): Promise { + if (this.abortPromise) { + await this.abortPromise; + return; + } if (this.terminal) return; this.terminal = true; + this.aborting = true; this.clearTimer(); this.pending = []; this.pendingBytes = 0; - this.writeChain = this.writeChain.then(async () => { + this.clearPayload(); + this.clearQueuedBatches(); + this.abortPromise = this.writeChain.then(async () => { try { // 已失效（disable 已清理 / halt 已让渡所有权）：不得再写 meta 覆盖新 owner if (this.disabled) return; @@ -340,7 +358,8 @@ export class ReplaySpool { this.release(); } }); - await this.writeChain; + this.writeChain = this.abortPromise; + await this.abortPromise; } /** 失效并删除条目（payload 超限 / Redis 不可用 / 冲刷异常等本 spool 自身的失败）。 */ @@ -360,31 +379,37 @@ export class ReplaySpool { this.pending = []; this.parts.length = 0; this.pendingBytes = 0; + this.clearQueuedBatches(); // 清理顺着 writeChain 串行：与 in-flight append 竞态时绝不出现「删除后又写回」 this.writeChain = this.writeChain.then(async () => { - if (deleteEntry) { - await this.store - .abortOwned( - this.identity.replayId, - this.ownerToken, - this.buildMeta("aborted", { abortReason: reason }) - ) - .catch(() => false); - } else { - // compare-delete 只删自己的 token：所有权已失时为安全 no-op - await this.store - .releaseOwner(this.identity.replayId, this.ownerToken) - .catch(() => undefined); + try { + if (deleteEntry) { + await this.store + .abortOwned( + this.identity.replayId, + this.ownerToken, + this.buildMeta("aborted", { abortReason: reason }) + ) + .catch(() => false); + } else { + // compare-delete 只删自己的 token：所有权已失时为安全 no-op + await this.store + .releaseOwner(this.identity.replayId, this.ownerToken) + .catch(() => undefined); + } + } finally { + this.release(); } }); logger.debug("[ReplaySpool] spool disabled", { replayId: this.identity.replayId.slice(0, 12), reason, }); - this.release(); } private released = false; + private aborting = false; + private abortPromise: Promise | null = null; private release(): void { if (this.released) return; @@ -409,6 +434,21 @@ export class ReplaySpool { this.clearFlushTimer(); this.clearOwnerHeartbeat(); } + + private takePayload(): string { + const payload = this.parts.join(""); + this.clearPayload(); + return payload; + } + + private clearPayload(): void { + this.parts.length = 0; + } + + private clearQueuedBatches(): void { + for (const batch of this.queuedBatches) batch.length = 0; + this.queuedBatches.clear(); + } } /** Forwarder 在 spool 创建前终止时，以 owner token 原子封死 Replay 条目。 */ diff --git a/src/lib/api/v1/_shared/request-body.ts b/src/lib/api/v1/_shared/request-body.ts index 60094b359..9013d5a93 100644 --- a/src/lib/api/v1/_shared/request-body.ts +++ b/src/lib/api/v1/_shared/request-body.ts @@ -3,10 +3,6 @@ import { createProblemResponse, normalizeZodPath } from "./error-envelope"; export type ParsedBodyResult = { ok: true; data: T } | { ok: false; response: Response }; -type JsonBodySchema = { - safeParse: (value: unknown) => { success: true; data: T } | { success: false; error: z.ZodError }; -}; - type ParseJsonBodyOptions = { validationErrorCode?: (error: z.ZodError) => string | undefined; }; @@ -20,10 +16,10 @@ type HonoJsonRequest = { }; }; -export async function parseJsonBody( +export async function parseJsonBody( request: Request, - schema: JsonBodySchema -): Promise> { + schema: S +): Promise>> { const contentType = request.headers.get("content-type") ?? ""; if (!contentType.toLowerCase().includes("application/json")) { return { @@ -73,11 +69,11 @@ export async function parseJsonBody( return { ok: true, data: parsed.data }; } -export async function parseHonoJsonBody( +export async function parseHonoJsonBody( c: HonoJsonRequest, - schema: JsonBodySchema, + schema: S, options?: ParseJsonBodyOptions -): Promise> { +): Promise>> { const contentType = c.req.header("content-type") ?? c.req.header("Content-Type") ?? diff --git a/src/lib/api/v1/schemas/audit-logs.ts b/src/lib/api/v1/schemas/audit-logs.ts index 5814d7386..14635b1a9 100644 --- a/src/lib/api/v1/schemas/audit-logs.ts +++ b/src/lib/api/v1/schemas/audit-logs.ts @@ -23,7 +23,9 @@ export const AuditLogListQuerySchema = z.object({ success: z .enum(["true", "false"]) .optional() - .transform((val) => (val === undefined ? undefined : val === "true")) + .transform((val: "true" | "false" | undefined) => + val === undefined ? undefined : val === "true" + ) .describe("Optional success filter."), from: IsoDateTimeStringSchema.optional().describe("Optional inclusive start time."), to: IsoDateTimeStringSchema.optional().describe("Optional inclusive end time."), diff --git a/src/lib/api/v1/schemas/me.ts b/src/lib/api/v1/schemas/me.ts index 4880d1240..39b3856a2 100644 --- a/src/lib/api/v1/schemas/me.ts +++ b/src/lib/api/v1/schemas/me.ts @@ -3,7 +3,7 @@ import { z } from "@hono/zod-openapi"; const NumberQuerySchema = z.coerce.number().optional(); const BooleanQuerySchema = z .union([z.literal("true"), z.literal("false"), z.boolean()]) - .transform((value) => value === true || value === "true") + .transform((value: "true" | "false" | boolean) => value === true || value === "true") .optional(); export const MeUsageLogsQuerySchema = z.object({ diff --git a/src/lib/api/v1/schemas/system-config.ts b/src/lib/api/v1/schemas/system-config.ts index 5dd0253a6..944e4cab2 100644 --- a/src/lib/api/v1/schemas/system-config.ts +++ b/src/lib/api/v1/schemas/system-config.ts @@ -28,7 +28,7 @@ const CodexPriorityBillingSourceSchema = z const TimeZoneSchema = z .string() .refine( - (value) => { + (value: string) => { try { new Intl.DateTimeFormat("en-US", { timeZone: value }); return true; diff --git a/src/lib/api/v1/schemas/usage-logs.ts b/src/lib/api/v1/schemas/usage-logs.ts index 2b01bc88a..6a63e3f67 100644 --- a/src/lib/api/v1/schemas/usage-logs.ts +++ b/src/lib/api/v1/schemas/usage-logs.ts @@ -3,7 +3,7 @@ import { z } from "@hono/zod-openapi"; const NumberQuerySchema = z.coerce.number().optional(); const BooleanQuerySchema = z .union([z.literal("true"), z.literal("false"), z.boolean()]) - .transform((value) => value === true || value === "true") + .transform((value: "true" | "false" | boolean) => value === true || value === "true") .optional(); export const UsageLogsQuerySchema = z.object({ diff --git a/tests/unit/proxy/billing-header-rectifier.test.ts b/tests/unit/proxy/billing-header-rectifier.test.ts index c9213ca1d..e4a973eb9 100644 --- a/tests/unit/proxy/billing-header-rectifier.test.ts +++ b/tests/unit/proxy/billing-header-rectifier.test.ts @@ -56,13 +56,12 @@ describe("rectifyBillingHeader", () => { }); test("system array with billing header mixed with real prompts - only removes billing header blocks", () => { - const message: Record = { - system: [ - { type: "text", text: "You are a helpful assistant." }, - { type: "text", text: "x-anthropic-billing-header: cc_version=2.1.36; cch=1;" }, - { type: "text", text: "Follow instructions carefully." }, - ], - }; + const originalSystem = [ + { type: "text", text: "You are a helpful assistant." }, + { type: "text", text: "x-anthropic-billing-header: cc_version=2.1.36; cch=1;" }, + { type: "text", text: "Follow instructions carefully." }, + ]; + const message: Record = { system: originalSystem }; const result = rectifyBillingHeader(message); @@ -72,6 +71,8 @@ describe("rectifyBillingHeader", () => { { type: "text", text: "You are a helpful assistant." }, { type: "text", text: "Follow instructions carefully." }, ]); + expect(message.system).not.toBe(originalSystem); + expect(originalSystem).toHaveLength(3); }); test("system as plain string that IS a billing header - deletes system field", () => { diff --git a/tests/unit/proxy/cache-ttl-override.test.ts b/tests/unit/proxy/cache-ttl-override.test.ts index 5a38e9e4a..3403bc3e5 100644 --- a/tests/unit/proxy/cache-ttl-override.test.ts +++ b/tests/unit/proxy/cache-ttl-override.test.ts @@ -42,20 +42,19 @@ describe("applyCacheTtlOverrideToMessage", () => { }); it("rewrites ttl on messages[].content[] ephemeral blocks (existing behavior)", () => { - const message: Record = { - messages: [ - { - role: "user", - content: [ - { - type: "text", - text: "hello", - cache_control: { type: "ephemeral" }, - }, - ], - }, - ], - }; + const originalMessages = [ + { + role: "user", + content: [ + { + type: "text", + text: "hello", + cache_control: { type: "ephemeral" }, + }, + ], + }, + ]; + const message: Record = { messages: originalMessages }; const applied = applyCacheTtlOverrideToMessage(message, "1h"); @@ -67,6 +66,8 @@ describe("applyCacheTtlOverrideToMessage", () => { type: "ephemeral", ttl: "1h", }); + expect(message.messages).not.toBe(originalMessages); + expect(originalMessages[0].content[0].cache_control).toEqual({ type: "ephemeral" }); }); it("rewrites both system and messages breakpoints in a single pass", () => { diff --git a/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts b/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts index 8d2d8948f..02cab8a5f 100644 --- a/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts +++ b/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts @@ -674,6 +674,34 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { expect(sessionState.currentModelRedirect.redirect.redirectedModel).toBe(fireworksRedirect); }); + test("shadow sessions share readonly request data while isolating top-level attempt state", () => { + const session = createSession(); + const requestBuffer = new ArrayBuffer(4 * 1024 * 1024); + session.request.buffer = requestBuffer; + + const createShadow = ( + ProxyForwarder as unknown as { + createStreamingShadowSession: (session: ProxySession, provider: Provider) => ProxySession; + } + ).createStreamingShadowSession; + const shadows = Array.from({ length: 4 }, (_, index) => + createShadow(session, createProvider({ id: index + 10, name: `p${index + 10}` })) + ); + + expect(shadows.every((shadow) => shadow.request.buffer === requestBuffer)).toBe(true); + expect(new Set(shadows.map((shadow) => shadow.request.buffer)).size).toBe(1); + + shadows[0].request.message.model = "shadow-only"; + + expect(session.request.message.model).not.toBe("shadow-only"); + expect(shadows[1].request.message.model).not.toBe("shadow-only"); + expect(shadows[0].request.message.messages).toBe(session.request.message.messages); + + shadows[0].request.buffer = new ArrayBuffer(16); + expect(session.request.buffer).toBe(requestBuffer); + expect(shadows[1].request.buffer).toBe(requestBuffer); + }); + test("switching to provider without redirect should clear stale redirect snapshot", () => { const requestedModel = "claude-haiku-4-5-20251001"; const fireworksRedirect = "accounts/fireworks/routers/kimi-k2p5-turbo"; diff --git a/tests/unit/proxy/replay-spool.test.ts b/tests/unit/proxy/replay-spool.test.ts index f3988350e..e366083d2 100644 --- a/tests/unit/proxy/replay-spool.test.ts +++ b/tests/unit/proxy/replay-spool.test.ts @@ -156,6 +156,13 @@ async function drainWriteChain(spool: ReplaySpool): Promise { await (spool as unknown as { writeChain: Promise }).writeChain; } +function retainedAsciiPartBytes(spool: ReplaySpool): number { + return (spool as unknown as { parts: string[] }).parts.reduce( + (total, part) => total + part.length, + 0 + ); +} + function makeOwnerSession(): ProxySession { return { replayState: { identity, ownerToken: "owner-token", role: "owner" }, @@ -365,9 +372,11 @@ describe("ReplaySpool：超尺寸自失效", () => { spool.observe(encoder.encode("x".repeat(32))); - // 计数同步归还；存储清理顺着 writeChain 串行执行（避免与 in-flight append 竞态） - expect(getActiveReplaySpoolCount()).toBe(0); + // 存储清理顺着 writeChain 串行执行；清理完成前继续占用 quota， + // 避免慢 Redis 下不断创建新 spool 绕过并发内存上限。 + expect(getActiveReplaySpoolCount()).toBe(1); await drainWriteChain(spool); + expect(getActiveReplaySpoolCount()).toBe(0); expect(storeControl.store.abortOwned).toHaveBeenCalledWith( identity.replayId, "owner-token", @@ -531,6 +540,76 @@ describe("ReplaySpool：completeAfterBilling 终态屏障", () => { expect(storeControl.store.completeOwned).toHaveBeenCalledTimes(1); }); + it("Redis 阻塞期间不提前复制 payload，PG 阻塞期间释放 parts", async () => { + const payloadBytes = 4 * 1024 * 1024; + const chunk = "x".repeat(64 * 1024); + let resolveRedis!: (value: number) => void; + let resolvePersist!: (value: "persisted") => void; + storeControl.store.writeOwned.mockImplementationOnce( + () => + new Promise((resolve) => { + resolveRedis = resolve; + }) + ); + storeControl.store.persistCompleted.mockImplementationOnce( + () => + new Promise<"persisted">((resolve) => { + resolvePersist = resolve; + }) + ); + const spool = makeSpool(); + for (let index = 0; index < 64; index += 1) { + spool.observe(encoder.encode(chunk)); + } + expect(retainedAsciiPartBytes(spool)).toBe(payloadBytes); + + const completion = spool.completeAfterBilling(10); + await vi.advanceTimersByTimeAsync(0); + + expect(storeControl.store.writeOwned).toHaveBeenCalledTimes(1); + expect(storeControl.store.persistCompleted).not.toHaveBeenCalled(); + expect(retainedAsciiPartBytes(spool)).toBe(payloadBytes); + + resolveRedis(1); + await vi.advanceTimersByTimeAsync(0); + + expect(storeControl.store.persistCompleted).toHaveBeenCalledTimes(1); + expect(storeControl.store.persistCompleted).toHaveBeenCalledWith( + expect.objectContaining({ + payload: "x".repeat(payloadBytes), + byteSize: payloadBytes, + }) + ); + expect(retainedAsciiPartBytes(spool)).toBe(0); + + resolvePersist("persisted"); + await completion; + }); + + it("payload 组装失败时封死热层并释放 heartbeat 与并发配额", async () => { + const spool = makeSpool(); + spool.observe(encoder.encode("data: partial\n\n")); + const parts = (spool as unknown as { parts: unknown[] }).parts; + parts[0] = { + toString: () => { + throw new Error("payload assembly failed"); + }, + }; + + await spool.completeAfterBilling(10); + + expect(storeControl.store.persistCompleted).not.toHaveBeenCalled(); + expect(storeControl.store.abortOwned).toHaveBeenCalledWith( + identity.replayId, + "owner-token", + expect.objectContaining({ status: "aborted", abortReason: "complete_failed" }) + ); + expect(getActiveReplaySpoolCount()).toBe(0); + + await vi.advanceTimersByTimeAsync(15_000); + expect(storeControl.store.renewOwnerLease).not.toHaveBeenCalled(); + }); + it("跨 chunk 截断的 UTF-8 序列在 complete 时冲刷解码尾部", async () => { const spool = makeSpool(); // "中" (0xE4 0xB8 0xAD) 只送前两字节：observe 阶段解码挂起，complete 时 flush 出替换字符 @@ -577,6 +656,125 @@ describe("ReplaySpool：abort 终态", () => { expect(getActiveReplaySpoolCount()).toBe(0); }); + it("abort 立即释放已累积的 payload", async () => { + const spool = makeSpool(); + spool.observe(encoder.encode("data: partial\n\n")); + + await spool.abort("upstream_error"); + + expect((spool as unknown as { parts: string[] }).parts).toEqual([]); + }); + + it("Redis flush 阻塞时 abort 立即释放 batch，并在 fenced cleanup 后释放并发配额", async () => { + let resolveRedis!: (value: number) => void; + storeControl.store.writeOwned.mockImplementationOnce( + (...args: unknown[]) => + new Promise((resolve) => { + resolveRedis = resolve; + void args; + }) + ); + const spool = makeSpool(); + spool.observe(encoder.encode("x".repeat(64 * 1024))); + await vi.advanceTimersByTimeAsync(0); + spool.observe(encoder.encode("y".repeat(64 * 1024))); + await vi.advanceTimersByTimeAsync(0); + + expect(storeControl.store.writeOwned).toHaveBeenCalledTimes(1); + const batch = storeControl.store.writeOwned.mock.calls[0][3] as string[]; + expect(batch.length).toBeGreaterThan(0); + const queuedBatches = (spool as unknown as { queuedBatches: Set }).queuedBatches; + expect(queuedBatches.size).toBe(2); + + let abortSettled = false; + const abortPromise = spool.abort("client_disconnect").then(() => { + abortSettled = true; + }); + await vi.advanceTimersByTimeAsync(0); + + expect(batch).toEqual([]); + expect([...queuedBatches].every((queuedBatch) => queuedBatch.length === 0)).toBe(true); + expect(abortSettled).toBe(false); + expect(getActiveReplaySpoolCount()).toBe(1); + expect(storeControl.store.abortOwned).not.toHaveBeenCalled(); + + resolveRedis(1); + await abortPromise; + expect(abortSettled).toBe(true); + expect(getActiveReplaySpoolCount()).toBe(0); + expect(storeControl.store.abortOwned).toHaveBeenCalledWith( + identity.replayId, + "owner-token", + expect.objectContaining({ status: "aborted", abortReason: "client_disconnect" }) + ); + }); + + it("bootstrap 阻塞时 abort 等待真正的 fenced cleanup 后再释放并发配额", async () => { + let resolveBootstrap!: (value: null) => void; + let resolveCleanup!: (value: boolean) => void; + storeControl.store.writeOwned.mockImplementationOnce( + () => + new Promise((resolve) => { + resolveBootstrap = resolve; + }) + ); + storeControl.store.abortOwned.mockImplementationOnce( + () => + new Promise((resolve) => { + resolveCleanup = resolve; + }) + ); + const spool = makeSpool(); + spool.bootstrap(); + await vi.advanceTimersByTimeAsync(0); + + let abortSettled = false; + const abortPromise = spool.abort("client_disconnect").then(() => { + abortSettled = true; + }); + await vi.advanceTimersByTimeAsync(0); + + resolveBootstrap(null); + await vi.advanceTimersByTimeAsync(0); + + expect(storeControl.store.abortOwned).toHaveBeenCalledTimes(1); + expect(abortSettled).toBe(false); + expect(getActiveReplaySpoolCount()).toBe(1); + + resolveCleanup(true); + await abortPromise; + expect(abortSettled).toBe(true); + expect(getActiveReplaySpoolCount()).toBe(0); + }); + + it("并发重复 abort 都等待同一个 fenced cleanup barrier", async () => { + let resolveRedis!: (value: number) => void; + storeControl.store.writeOwned.mockImplementationOnce( + () => + new Promise((resolve) => { + resolveRedis = resolve; + }) + ); + const spool = makeSpool(); + spool.observe(encoder.encode("x".repeat(64 * 1024))); + await vi.advanceTimersByTimeAsync(0); + + const firstAbort = spool.abort("client_disconnect"); + let secondAbortSettled = false; + const secondAbort = spool.abort("client_disconnect").then(() => { + secondAbortSettled = true; + }); + await vi.advanceTimersByTimeAsync(0); + + const secondSettledBeforeCleanup = secondAbortSettled; + resolveRedis(1); + await Promise.all([firstAbort, secondAbort]); + + expect(secondSettledBeforeCleanup).toBe(false); + expect(getActiveReplaySpoolCount()).toBe(0); + expect(storeControl.store.abortOwned).toHaveBeenCalledTimes(1); + }); + it("abort 后 observe 与 complete 均无副作用", async () => { const spool = makeSpool(); await spool.abort("client_disconnect"); From 3fe3225c9f6397d22db27193199e2a8fef4a05f7 Mon Sep 17 00:00:00 2001 From: Ding <44717411+ding113@users.noreply.github.com> Date: Mon, 10 Aug 2026 17:40:06 +0800 Subject: [PATCH 07/12] fix(proxy): recognize terminal Responses compaction streams (#1410) (#1411) * fix(stream-gate): recognize Responses compaction in terminal frames Some Responses upstreams return compaction output only in response.completed without first emitting response.output_item.done, so the content gate treated the stream as empty and triggered false failover with circuit-breaking. classifyParsedFrame now accepts the protocol family and, for openai-responses, detects a compaction output item with non-empty encrypted_content inside response.completed, classifying the frame as content so the gate commits the stream. Regression tests cover compaction-only terminal frames and custom tool-call input deltas across the classifier, content gate, and forwarder integration paths. Fixes #1410 * fix(stream-gate): require non-empty string for compaction encrypted_content The compaction signal rule on response.output_item.done accepted any truthy encrypted_content value, allowing non-string types (booleans, numbers, objects) to be misclassified as content. Consolidate the per-item type guard into isNonEmptyCompactionItem and apply it to both response.output_item.done and response.completed paths so the opaque state must be a non-empty string before a frame is committed as content. Add regression tests covering malformed encrypted_content types across event variants. --- .../proxy/stream-gate/frame-classifier.ts | 47 +++++++-- .../proxy/stream-gate-content-gate.test.ts | 36 +++++++ .../stream-gate-forwarder-integration.test.ts | 76 +++++++++++++++ .../stream-gate-frame-classifier.test.ts | 96 +++++++++++++++++++ 4 files changed, 247 insertions(+), 8 deletions(-) diff --git a/src/app/v1/_lib/proxy/stream-gate/frame-classifier.ts b/src/app/v1/_lib/proxy/stream-gate/frame-classifier.ts index b6dac5c55..609616b6a 100644 --- a/src/app/v1/_lib/proxy/stream-gate/frame-classifier.ts +++ b/src/app/v1/_lib/proxy/stream-gate/frame-classifier.ts @@ -167,12 +167,6 @@ const STREAM_SIGNALS: Record = { eventTypes: ["response.code_interpreter_call_code.done"], anyPaths: ["code"], }, - { - // Remote/server-side compaction 的 opaque state 是完整协议 payload，只有非空且类型精确匹配才提交。 - eventTypes: ["response.output_item.done"], - anyPaths: ["item.encrypted_content"], - valueMatches: [{ path: "item.type", values: ["compaction"] }], - }, { // output_item.added 的 name/id/status 只是结构元数据；真实 payload 到达前不能提交， // 否则紧随其后的 response.failed / 断流将失去透明 fallback 机会。 @@ -356,7 +350,7 @@ function classifyFrameInner( return "malformed"; } - const outerVerdict = classifyParsedFrame(signal, eventName, parsed); + const outerVerdict = classifyParsedFrame(family, signal, eventName, parsed); if (outerVerdict !== "neutral" || family !== "gemini" || Array.isArray(parsed)) { return outerVerdict; } @@ -365,11 +359,12 @@ function classifyFrameInner( // 只有外层中性时才解包，供所有门控与 observer 共用同一分类结果。 const response = (parsed as Record).response; return response && typeof response === "object" && !Array.isArray(response) - ? classifyParsedFrame(signal, eventName, response) + ? classifyParsedFrame(family, signal, eventName, response) : outerVerdict; } function classifyParsedFrame( + family: ProtocolFamily, signal: StreamSignal, eventName: string | null, parsed: object @@ -385,6 +380,9 @@ function classifyParsedFrame( for (const rule of signal.errorRules) { if (frameRuleMatches(rule, effective, parsed)) return "error"; } + if (family === "openai-responses" && isResponsesCompactionContent(effective, parsed)) { + return "content"; + } for (const rule of signal.contentRules) { if (frameRuleMatches(rule, effective, parsed)) return "content"; } @@ -397,6 +395,39 @@ function classifyParsedFrame( return "neutral"; } +/** + * Remote/server-side compaction 的 opaque state 是完整协议 payload。部分上游只在 + * response.completed 中返回 output, 不会先发送 response.output_item.done。 + */ +function isResponsesCompactionContent(eventType: string, parsed: object): boolean { + if (Array.isArray(parsed)) return false; + + const record = parsed as Record; + if (eventType === "response.output_item.done") { + return isNonEmptyCompactionItem(record.item); + } + if (eventType !== "response.completed") return false; + + const response = record.response; + if (response === null || typeof response !== "object" || Array.isArray(response)) return false; + + const output = (response as Record).output; + if (!Array.isArray(output)) return false; + + return output.some(isNonEmptyCompactionItem); +} + +/** 同一 output item 内的 type 与 opaque state 必须同时满足协议类型约束。 */ +function isNonEmptyCompactionItem(item: unknown): boolean { + if (item === null || typeof item !== "object" || Array.isArray(item)) return false; + const record = item as Record; + return ( + record.type === "compaction" && + typeof record.encrypted_content === "string" && + record.encrypted_content !== "" + ); +} + /** 单条帧规则 AND 语义；空规则永不命中（防目录笔误把所有帧判成内容/错误）。 */ function frameRuleMatches(rule: FrameRule, eventType: string, parsed: unknown): boolean { if (rule.eventTypes && rule.eventTypes.length > 0 && !rule.eventTypes.includes(eventType)) { diff --git a/tests/unit/proxy/stream-gate-content-gate.test.ts b/tests/unit/proxy/stream-gate-content-gate.test.ts index bd97a540a..56140dd01 100644 --- a/tests/unit/proxy/stream-gate-content-gate.test.ts +++ b/tests/unit/proxy/stream-gate-content-gate.test.ts @@ -254,6 +254,42 @@ describe("runStreamContentGate", () => { expect(new TextDecoder().decode(rest.value)).toBe(completed); }); + it("openai-responses: commits compaction carried only by response.completed", async () => { + const completed = + 'event: response.completed\ndata: {"type":"response.completed","response":{"status":"completed","output":[{"type":"compaction","encrypted_content":"opaque-state"}]}}\n\n'; + const reader = readerFromChunks([completed]); + + const result = await runStreamContentGate(reader, { + ...GATE_OPTIONS, + family: "openai-responses", + }); + + expect(result.committed).toBe(true); + if (!result.committed) return; + expect(await drainPrefix(result.prefixChunks)).toBe(completed); + expect(result.readerDone).toBe(false); + }); + + it("openai-responses: commits custom tool-call input before response.completed", async () => { + const toolInput = + 'event: response.custom_tool_call_input.delta\ndata: {"type":"response.custom_tool_call_input.delta","delta":"{\\"path\\":\\"README.md\\"}"}\n\n'; + const completed = + 'event: response.completed\ndata: {"type":"response.completed","response":{"status":"completed"}}\n\n'; + const reader = readerFromChunks([toolInput, completed]); + + const result = await runStreamContentGate(reader, { + ...GATE_OPTIONS, + family: "openai-responses", + }); + + expect(result.committed).toBe(true); + if (!result.committed) return; + expect(await drainPrefix(result.prefixChunks)).toBe(toolInput); + expect(result.readerDone).toBe(false); + const rest = await reader.read(); + expect(new TextDecoder().decode(rest.value)).toBe(completed); + }); + it("gemini: usage-only chunks buffer until content commits", async () => { const reader = readerFromChunks([ 'data: {"usageMetadata":{"totalTokenCount":1}}\n\n', diff --git a/tests/unit/proxy/stream-gate-forwarder-integration.test.ts b/tests/unit/proxy/stream-gate-forwarder-integration.test.ts index 8ce3c6296..23d4beb42 100644 --- a/tests/unit/proxy/stream-gate-forwarder-integration.test.ts +++ b/tests/unit/proxy/stream-gate-forwarder-integration.test.ts @@ -224,6 +224,35 @@ const OPENAI_RESPONSES_WINNER_FRAMES = [ }), ]; +const VALID_OPENAI_RESPONSES_STREAMS = [ + { + name: "terminal compaction output", + frames: [ + sseFrame("response.completed", { + type: "response.completed", + response: { + id: "resp_compaction", + status: "completed", + output: [{ id: "cmp_1", type: "compaction", encrypted_content: "opaque-state" }], + }, + }), + ], + }, + { + name: "custom tool-call input deltas", + frames: [ + sseFrame("response.custom_tool_call_input.delta", { + type: "response.custom_tool_call_input.delta", + delta: '{"path":"README.md"}', + }), + sseFrame("response.completed", { + type: "response.completed", + response: { id: "resp_tool", status: "completed" }, + }), + ], + }, +] as const; + type ReplayGateCase = { name: string; providerType: Provider["providerType"]; @@ -516,6 +545,30 @@ describe("F1 stream content gate x ProxyForwarder sequential path", () => { expect(mocks.recordFailure).not.toHaveBeenCalled(); }); + test("Responses terminal compaction 在 enforce 模式下直接提交且不计入熔断", async () => { + const provider = createProvider({ id: 1, name: "compaction", providerType: "codex" }); + const session = createSession(); + session.setProvider(provider); + Object.assign(session, { + requestUrl: new URL("https://example.com/v1/responses"), + originalFormat: "response", + endpointPolicy: resolveEndpointPolicy("/v1/responses"), + }); + + const frames = VALID_OPENAI_RESPONSES_STREAMS[0].frames.slice(); + const doForward = spyOnDoForward(); + doForward.mockImplementationOnce(async () => createSseResponse(frames)); + + const response = await ProxyForwarder.send(session); + const text = await response.text(); + + expect(response.status).toBe(200); + expect(text).toBe(frames.join("")); + expect(doForward).toHaveBeenCalledTimes(1); + expect(mocks.pickRandomProviderWithExclusion).not.toHaveBeenCalled(); + expect(mocks.recordFailure).not.toHaveBeenCalled(); + }); + test("terminal-only 流（message_stop 即终止）按 empty_stream 失败并切换供应商", async () => { const provider1 = createProvider({ id: 1, name: "gate-p1" }); const provider2 = createProvider({ id: 2, name: "gate-p2" }); @@ -631,6 +684,29 @@ describe("F1 stream content gate x ProxyForwarder sequential path", () => { expect(mocks.recordFailure).not.toHaveBeenCalled(); }); + test.each(VALID_OPENAI_RESPONSES_STREAMS)( + "Replay owner 将 $name 视为有效内容，不触发 502/failover/熔断", + async ({ frames }) => { + const provider = createProvider({ id: 1, name: "responses-valid", providerType: "codex" }); + const session = createSession(); + session.setProvider(provider); + attachReplayOwner(session, REPLAY_GATE_CASES[0]); + + const streamFrames = frames.slice(); + const doForward = spyOnDoForward(); + doForward.mockImplementationOnce(async () => createSseResponse(streamFrames)); + + const response = await ProxyForwarder.send(session); + const text = await response.text(); + + expect(response.status).toBe(200); + expect(text).toBe(streamFrames.join("")); + expect(doForward).toHaveBeenCalledTimes(1); + expect(mocks.pickRandomProviderWithExclusion).not.toHaveBeenCalled(); + expect(mocks.recordFailure).not.toHaveBeenCalled(); + } + ); + test("Replay owner 在所有 precommit attempt 失败后立即释放所有权", async () => { const provider = createProvider({ id: 1, name: "replay-only", providerType: "codex" }); const session = createSession(); diff --git a/tests/unit/proxy/stream-gate-frame-classifier.test.ts b/tests/unit/proxy/stream-gate-frame-classifier.test.ts index 3e8fb3053..50b8ecde8 100644 --- a/tests/unit/proxy/stream-gate-frame-classifier.test.ts +++ b/tests/unit/proxy/stream-gate-frame-classifier.test.ts @@ -270,6 +270,40 @@ describe("classifyFrame: openai-responses", () => { ).toBe("content"); }); + it("content: custom tool-call input delta and done payloads", () => { + expect( + classifyFrame( + "openai-responses", + "response.custom_tool_call_input.delta", + '{"type":"response.custom_tool_call_input.delta","delta":"{\\"path\\":\\"README.md\\"}"}' + ) + ).toBe("content"); + expect( + classifyFrame( + "openai-responses", + "response.custom_tool_call_input.done", + '{"type":"response.custom_tool_call_input.done","input":"{\\"path\\":\\"README.md\\"}"}' + ) + ).toBe("content"); + }); + + it("neutral: empty custom tool-call input delta and done payloads", () => { + expect( + classifyFrame( + "openai-responses", + "response.custom_tool_call_input.delta", + '{"type":"response.custom_tool_call_input.delta","delta":""}' + ) + ).toBe("neutral"); + expect( + classifyFrame( + "openai-responses", + "response.custom_tool_call_input.done", + '{"type":"response.custom_tool_call_input.done","input":""}' + ) + ).toBe("neutral"); + }); + it("neutral: output_item.added carrying only tool metadata", () => { expect( classifyFrame( @@ -300,6 +334,68 @@ describe("classifyFrame: openai-responses", () => { ).toBe("content"); }); + it("content: response.completed carrying compaction output with opaque state", () => { + expect( + classifyFrame( + "openai-responses", + "response.completed", + '{"type":"response.completed","response":{"status":"completed","output":[{"type":"compaction","encrypted_content":"opaque-state"}]}}' + ) + ).toBe("content"); + }); + + it("terminal: response.completed without a non-empty compaction output", () => { + expect( + classifyFrame( + "openai-responses", + "response.completed", + '{"type":"response.completed","response":{"status":"completed","output":[{"type":"compaction","encrypted_content":""}]}}' + ) + ).toBe("terminal"); + expect( + classifyFrame( + "openai-responses", + "response.completed", + '{"type":"response.completed","response":{"status":"completed","output":[{"type":"reasoning","encrypted_content":"opaque-state"}]}}' + ) + ).toBe("terminal"); + expect( + classifyFrame( + "openai-responses", + "response.completed", + '{"type":"response.completed","response":{"status":"completed","output":[{"type":"compaction","encrypted_content":""},{"type":"reasoning","encrypted_content":"opaque-state"}]}}' + ) + ).toBe("terminal"); + }); + + it("rejects non-string compaction encrypted content", () => { + for (const encryptedContent of [true, 42, { opaque: "state" }]) { + expect( + classifyFrame( + "openai-responses", + "response.output_item.done", + JSON.stringify({ + type: "response.output_item.done", + item: { type: "compaction", encrypted_content: encryptedContent }, + }) + ) + ).toBe("neutral"); + expect( + classifyFrame( + "openai-responses", + "response.completed", + JSON.stringify({ + type: "response.completed", + response: { + status: "completed", + output: [{ type: "compaction", encrypted_content: encryptedContent }], + }, + }) + ) + ).toBe("terminal"); + } + }); + it("neutral: empty or non-compaction encrypted output item", () => { expect( classifyFrame( From f01f9f87f91c4f2875803d8a141d1301ef167e50 Mon Sep 17 00:00:00 2001 From: "AptS:1547" Date: Tue, 11 Aug 2026 19:18:13 +0800 Subject: [PATCH 08/12] Merge pull request #1414 from ding113/fix/issue-1408-replay-oom fix(proxy): bound Replay disconnect memory retention (#1408) --- .env.example | 2 + CHANGELOG.md | 11 + docs/troubleshooting/issue-1408-replay-oom.md | 285 ++++++++++ src/app/v1/_lib/proxy/replay/replay-spool.ts | 118 ++-- src/app/v1/_lib/proxy/response-handler.ts | 71 ++- src/lib/config/env.schema.ts | 7 + .../session-manager-detail-snapshots.test.ts | 63 +++ src/lib/session-manager.ts | 81 ++- tests/load/issue-1408-replay-oom/README.md | 96 ++++ .../drive-disconnect-waves.cjs | 216 ++++++++ .../issue-1408-replay-oom/memory-probe.cjs | 41 ++ .../issue-1408-replay-oom/mock-upstream.cjs | 158 ++++++ tests/load/issue-1408-replay-oom/run-wave.sh | 51 ++ .../issue-1408-replay-oom/sample-container.sh | 50 ++ .../start-mock-container.sh | 54 ++ .../env-store-session-response-body.test.ts | 28 + .../lib/session-manager-redaction.test.ts | 36 ++ .../proxy/issue-1408-load-fixture.test.ts | 520 ++++++++++++++++++ tests/unit/proxy/replay-spool.test.ts | 263 +++++++-- .../response-handler-stream-terminal.test.ts | 168 +++++- 20 files changed, 2189 insertions(+), 130 deletions(-) create mode 100644 docs/troubleshooting/issue-1408-replay-oom.md create mode 100644 tests/load/issue-1408-replay-oom/README.md create mode 100644 tests/load/issue-1408-replay-oom/drive-disconnect-waves.cjs create mode 100644 tests/load/issue-1408-replay-oom/memory-probe.cjs create mode 100644 tests/load/issue-1408-replay-oom/mock-upstream.cjs create mode 100755 tests/load/issue-1408-replay-oom/run-wave.sh create mode 100755 tests/load/issue-1408-replay-oom/sample-container.sh create mode 100755 tests/load/issue-1408-replay-oom/start-mock-container.sh create mode 100644 tests/unit/proxy/issue-1408-load-fixture.test.ts diff --git a/.env.example b/.env.example index b6c6d72b8..cb3fd0dfb 100644 --- a/.env.example +++ b/.env.example @@ -105,6 +105,8 @@ STORE_SESSION_RESPONSE_BODY=true # 是否在 Redis 中存储会话响应 # - true：存储（SSE/JSON），用于调试/定位问题（Redis 临时缓存） # - false：不存储响应体（注意：不影响本次请求处理；仅影响后续查看 response body） # 说明：该开关不影响内部统计读取响应体（tokens/费用统计、SSE 假 200 检测仍会进行） +SESSION_RESPONSE_BODY_MAX_BYTES=5242880 # 单份会话响应体 Redis 存储上限（默认 5 MiB，范围 64 KiB-64 MiB） + # 超限正文不落 Redis；before/after snapshot 的 headers/meta 仍保留 # Dashboard 配置 DASHBOARD_LOGS_POLL_INTERVAL_MS=5000 # 日志页自动刷新轮询间隔（毫秒，默认 5000，范围 250-60000） diff --git a/CHANGELOG.md b/CHANGELOG.md index d5474c884..aa6aca9c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,17 @@ --- +## Unreleased + +### 修复 + +- 修复 Replay owner 在客户端断线后保留完整流正文和 300 秒传输资源导致的内存失控：限制 Redis + write-behind backlog，Replay 失效后按断线起点恢复 60 秒 drain，并为 Redis session response body + 增加默认 5 MiB 的可配置存储上限，避免大 SSE 正文及 before/after 快照放大内存和持久化压力； + 三份 response body 的物理存储去重由 #1415 跟踪 (#1408) + +--- + ## v0.8.7 (2026-06-14) ### 新增 diff --git a/docs/troubleshooting/issue-1408-replay-oom.md b/docs/troubleshooting/issue-1408-replay-oom.md new file mode 100644 index 000000000..57f9b03ff --- /dev/null +++ b/docs/troubleshooting/issue-1408-replay-oom.md @@ -0,0 +1,285 @@ +# Issue #1408 Replay 断线流内存失控根因报告 + +## 调查结论 + +Issue #1408 的 Node 内存失控触发链已经在本地完成机制级复现和因子隔离。该链路能够在受控 +环境中稳定触发同类 V8 heap OOM，并与生产环境的 Replay 配置、客户端断线和 Redis 压力现象 +高度重合。 + +已经证明的主触发链由两个同时存在的生命周期缺陷组成： + +1. `ReplaySpool` 在活跃响应期间把每个流块解码成字符串，同时保存在本地 + `parts[]`、待写 `pending`/`writeChain` batch 和 Redis LIST 中。本地 `parts[]` 会一直保留 + 整条响应，直到上游出现终态或 Replay 被禁用。 +2. Replay owner 的客户端断开后，`ProxyResponseHandler` 把普通 60 秒 drain 窗口改成 + `REPLAY_MAX_DETACHED_MS`，默认 300 秒。spool 后续失效时，这个已选定的窗口不会降级， + 因而失去 Replay 价值的上游流仍可能被保留到 300 秒。 + +这两点使断线流的 JS 字符串、ArrayBuffer、Undici Response、后台任务和 socket 在同一个 +300 秒窗口内按请求波次叠加。持续到来的断线请求不需要形成永久引用泄漏，也可以在最早一批 +请求进入超时回收之前耗尽 V8 heap 或容器内存。 + +生产故障发生时没有 heap snapshot，因此本报告证明的是“存在一条足以解释并复现 #1408 的 +决定性机制”，不是从生产进程对象图中证明了唯一根因。shadow/hedge 请求复制等其他 v0.9.x +内存放大路径仍可能在实际流量中共同贡献。 + +## 适用版本与代码边界 + +- 有效复现版本：`v0.9.2`，提交 `ccbad37f266e3e69d57a4427e2f27cf288796e63` +- 修复目标分支：`dev`，调查时提交 `3fe3225c9f6397d22db27193199e2a8fef4a05f7` +- `/v1/responses` 真实 Codex 转换路径 +- `ENABLE_REQUEST_REPLAY=true` +- `REPLAY_MAX_PAYLOAD_BYTES=8 MiB` +- `REPLAY_MAX_DETACHED_MS=300000` +- `STREAM_GATE_MODE=enforce` + +PR #1405 已补充 queued batch/abort 清理与 request copy-on-write，但 `dev` 中仍保留整条 +`ReplaySpool.parts[]`，也没有处理 spool 失效后的 drain 窗口降级。因此 #1405 降低了部分 +异常路径的保留风险，但没有覆盖本报告复现的主触发链。 + +## 本地复现夹具 + +隔离环境使用独立 PostgreSQL、Redis、mock upstream 和两个 v0.9.2 app 容器。app 容器限制为 +1 GiB，避免实验影响扩散到其他进程。 + +mock upstream 对每个请求发送有效的 `response.output_text.delta` SSE 帧，累计约 7.5 MiB 后 +保持连接打开且不发送终态。客户端确认 mock 收到请求后约 250 ms 主动断开。 + +可重复运行的夹具已纳入仓库： + +```text +tests/load/issue-1408-replay-oom/mock-upstream.cjs +tests/load/issue-1408-replay-oom/drive-disconnect-waves.cjs +tests/load/issue-1408-replay-oom/memory-probe.cjs +tests/load/issue-1408-replay-oom/sample-container.sh +tests/load/issue-1408-replay-oom/run-wave.sh +tests/load/issue-1408-replay-oom/start-mock-container.sh +tests/load/issue-1408-replay-oom/README.md +``` + +本次调查的原始采样与 fatal report 保留在本机： + +```text +/private/tmp/cch1408-wave-on-samples.out +/private/tmp/cch1408-wave-off-samples.out +/private/tmp/cch1408-wave-64k-samples.out +/private/tmp/cch1408-wave-fixed2-samples.out +/private/tmp/cch1408-wave-fixed2-repeat2-samples.out +/private/tmp/cch1408-on-reports/report.20260811.072324.1.0.001.json +``` + +## 证据一：Replay 把断线 drain 从 60 秒延长到 300 秒 + +固定 8 个断线请求时： + +| 场景 | 25 秒 | 60 秒 | 300 秒后 | +| --- | --- | --- | --- | +| Replay off | external 79.62 MiB，ArrayBuffer 75.65 MiB，21 sockets | 8/8 timeout 开始释放 | external 4.73 MiB，ArrayBuffer 0.77 MiB，13 sockets | +| Replay on | external 81.10 MiB，ArrayBuffer 77.13 MiB，21 sockets | 0/8 timeout，继续保持 | 8/8 timeout 后释放 | + +Replay-on 在终态清理后的最终状态为： + +```text +external 4.67 MiB +arrayBuffers 0.70 MiB +TCP sockets 13 +Async tasks 0 +``` + +这说明单波请求最终会释放，但 300 秒窗口允许多个请求波次在释放前持续叠加。 + +## 证据二：40 个活跃 Replay 流复现 V8 heap OOM + +以 10 秒间隔发送 5 波、每波 8 个不同 Replay 请求。所有请求均在客户端断开后保持上游悬挂。 + +| 活跃任务 | heapUsed | external | ArrayBuffer | RSS | +| ---: | ---: | ---: | ---: | ---: | +| 8 | 193.78 MiB | 92.27 MiB | 88.30 MiB | 383.68 MiB | +| 16 | 258.36 MiB | 168.32 MiB | 164.35 MiB | 520.14 MiB | +| 24 | 315.20 MiB | 240.16 MiB | 236.19 MiB | 664.54 MiB | +| 32 | 380.65 MiB | 314.59 MiB | 310.62 MiB | 811.62 MiB | +| 40 | 382.60 MiB | 311.58 MiB | 307.61 MiB | 793.50 MiB | + +随后 Node 进程直接输出： + +```text +FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory +``` + +容器退出码为 `133`。Node fatal report 记录： + +```text +javascriptHeap.usedMemory 437,947,120 bytes +javascriptHeap.memoryLimit 562,036,736 bytes +javascriptHeap.externalMemory 369,633,015 bytes +javascriptHeap.heapSpaces.old_space.used 408,208,280 bytes +resourceUsage.rss 904,556,544 bytes +resourceUsage.maxRss 929,734,656 bytes +``` + +这与 #1408 的生产现象属于同一种 Node 内存增长并最终退出的故障类别。本地容器有 1 GiB 限制， +所以在约 0.9 GiB RSS 时提前终止；生产容器没有内存限制，允许同一类请求波次继续累积到更高 +RSS。生产环境约 30 GiB 的绝对值不能仅由该缩小夹具外推。 + +## 证据三：64 KiB spool 上限隔离出 `parts[]` heap 主因 + +第三组实验保持 Replay 和 300 秒 drain 不变，只把 `REPLAY_MAX_PAYLOAD_BYTES` 降到 64 KiB。 +每个 spool 在首个大块后立即 `payload_too_large`，清空本地 `parts[]`，但上游流仍按 300 秒 +窗口 drain。 + +相同 8 波、共 64 个活跃断线请求的结果： + +```text +active stream tasks 64 +heapUsed 133.46 MiB +external 586.72 MiB +arrayBuffers 582.75 MiB +RSS 886.67 MiB +container running +``` + +正常 8 MiB spool 在 40 个活跃流时 `heapUsed` 已到 382.60 MiB 并触发 V8 fatal OOM；64 KiB +spool 在 64 个活跃流时 `heapUsed` 仍约 133 MiB。唯一关键变量是 spool 本地正文是否继续保留。 +因此 `ReplaySpool.parts[]` 是本次 V8 heap OOM 的决定性持有对象，300 秒 detached drain 是并发 +驻留时间放大器。 + +## 证据四：Session Response Body 三份复制放大 Redis + +第一版 Node 修复移除 `parts[]` 并缩短失效 spool 的 drain 后，Node 已不再触发 V8 OOM，但隔离 +Redis 在约 300 秒 RDB 保存点仍退出 `137`。检查 Redis key 和配置后确认： + +- `STORE_SESSION_RESPONSE_BODY=true`； +- 本夹具约 7.5 MiB 的 SSE 小于 `STREAM_STATS_MAX_BUFFER_BYTES=10 MiB`，会形成完整统计快照； +- 同一正文写入 legacy/request response、before snapshot、after snapshot 三份 Redis key； +- 64 个请求约产生 `3 x 64 x 7.5 MiB = 1.4 GiB` 的正文值。 + +第一版修复 Redis 的实际峰值为： + +```text +used_memory_human 1.50G +used_memory_peak_human 1.51G +OOMKilled=true +ExitCode=137 +``` + +旧版应用 fatal OOM 发生在 Redis 退出之前，因此 Redis 退出不是 Node fatal 的起因；它是独立的 +伴随放大器。RDB/AOF fork 与写入压力会进一步放大整机内存和 I/O 压力，这与 #1408 中 Redis +`bio_aof` 阻塞、healthcheck 超时的后续现象一致。 + +## 代码持有链 + +当前路径可以简化为： + +```text +upstream Uint8Array + -> ResponseHandler.observeChunk() + -> BoundedStreamTextAccumulator / stream transport buffers + -> ReplaySpool.observe() + -> TextDecoder string + -> pending[] + -> queued writeChain batch + -> parts[] retained until terminal + -> Redis LIST copy + +client disconnect + -> responsePump.startDrain() + -> replay owner selects 300s timeout + -> each new request wave adds another retained response + -> V8 old_space reaches heap limit before oldest wave expires + +stream finalization + -> storeSessionResponse() + -> before response snapshot + -> after response snapshot + -> three Redis values retain the same multi-MiB SSE body until SESSION_TTL + -> RDB/AOF persistence amplifies Redis memory and I/O pressure +``` + +spool 因 payload 超限、Redis 异常或 owner lease 丢失而失效时，会清理自身正文，但 +`ProxyResponseHandler` 已经选择的 300 秒 timer 仍继续运行，所以失效后的 Response、ArrayBuffer、 +socket 和后台任务仍会保留。这解释了 8 MiB 超限样本在 spool 清空后依然保持约 77 MiB +ArrayBuffer 到 300 秒的现象。 + +## 已实现修复 + +修复在 `dev` 提交 `3fe3225c9f6397d22db27193199e2a8fef4a05f7` 的工作树上完成，包含： + +1. 删除 `ReplaySpool.parts[]`，活跃正文只长期保存在 Redis fenced chunks 中。 +2. 完成时从 Redis 回读 chunks，校验数量后重建 durable payload；大 payload 的回读、拼接和 PG + 持久化全局串行，避免多个终态同时形成 heap 峰值。 +3. Redis write-behind backlog 上限固定为 1 MiB。超过时以 + `write_backlog_too_large` fail-open 关闭当前 spool，并立即清空 pending/queued batch。 +4. spool disable、halt 或 abort 通过一次性 `onInactive` 通知 response handler；若客户端已断开， + drain 从 Replay 300 秒降回普通 60 秒，并从实际断线时刻计算剩余时间。 +5. 新增 `SESSION_RESPONSE_BODY_MAX_BYTES`，默认 5 MiB、范围 64 KiB 到 64 MiB。legacy response + 和 before/after snapshot 都按 UTF-8 字节限制；超限时删除同 key 的旧正文，但继续保存 + headers/meta。三份 response body 的物理存储去重由 #1415 跟踪。 +6. 保留 Replay fenced owner、终态计费屏障、live attach、PG durable winner 和冲突处理语义。 + +## 修复负载回归 + +修复镜像保持相同 PostgreSQL、provider、key、mock、Replay 配置和 1 GiB app cgroup，连续运行 +两组完整 64 请求波次，另有一组 8 请求预检，共 136 个断线请求。 + +第一组关键点： + +| 活跃任务 | heapUsed | external | ArrayBuffer | RSS | +| ---: | ---: | ---: | ---: | ---: | +| 40 | 87.97 MiB | 371.74 MiB | 367.77 MiB | 594.50 MiB | +| 48 | 87.86 MiB | 442.92 MiB | 438.95 MiB | 679.63 MiB | + +第二组在复用同一进程和 allocator 状态后，48 个活跃任务时 `heapUsed=92.01 MiB`；整个波次 +最高观测 `heapUsed=155.31 MiB`、`RSS=905.63 MiB`，随后 64 个任务全部清理，`heapUsed` 回到 +约 95 MiB。对比旧版 40 个任务时 `heapUsed=382.60 MiB` 并 fatal OOM，Node heap 持有链已经 +被切断。继续静默等待 GC 后，进程为 `heapUsed=83.38 MiB`、`external=4.64 MiB`、 +`ArrayBuffer=0.67 MiB`、13 sockets，证明第二轮峰值没有形成阶梯式引用累积。 + +累计运行日志： + +```text +Client abort drain window exceeded 136 +write_backlog_too_large 136 +oversized session body skipped 408 +FATAL ERROR / heap out of memory 0 +remaining async tasks 0 +``` + +Redis 在两轮后： + +```text +used_memory_human 2.87M +used_memory_peak_human 5.34M +rdb_saves 3 +rdb_last_bgsave_status ok +rdb_last_cow_size 1138688 +container running, oom=false, exit=0 +``` + +上述负载回归显式使用 1 MiB session body 边界，证明它消除了原先约 1.50 GiB 的三份正文驻留， +并已跨过 `save 300 100` 的 RDB fork 点。当前产品默认值为 5 MiB；1 MiB 到 5 MiB 正文仍可能 +形成三份 Redis value，该放大边界及 5 MiB 负载/RDB 验证由 #1415 跟踪，不属于上述实验已证明的范围。 + +## 测试与证据边界 + +focused 回归共 5 个文件、104 个测试，覆盖： + +- 64 KiB flush、1 MiB backlog 包含边界和超限清理； +- Redis/PG 阻塞、abort/disable/halt 竞态、幂等和 active spool 配额释放； +- Redis chunks 缺失、durable 冲突、并发完成串行化和 UTF-8 截断尾部； +- spool 在断线前/后失效，以及活跃 spool 保持完整 300 秒窗口； +- session body 默认值、64 KiB/64 MiB 配置边界、UTF-8 字节边界、旧值删除； +- 超限 snapshot 只删除 body，headers/meta 继续保留。 + +最终 checkout 已通过 `bun run lint:fix`、`bun run lint`、`bun run typecheck` 和宿主机 +`bun run build`。Biome 仅提示配置 schema URL 为 2.5.6、CLI 为 2.5.7，没有 lint 错误或自动改动。 + +最终全量 `bun run test` 仍只有一个失败。唯一失败是既有 `language-switcher` sessionStorage +console 断言，隔离复跑仍为相同失败，与本次代理、Replay 和 session 存储路径无关。全量并行运行 +另报告一次 `price-list-ui-requirements` worker teardown console RPC rejection；该文件隔离复跑 4/4 +通过，因此记录为测试 harness 并行 teardown 噪声，不计入本次修复通过项。 + +## 调查状态 + +本地机制复现、修复、重复负载和 Redis 持久化边界均已完成。结论是“已证明并修复一条足以复现 +`#1408` 的决定性 Replay/断线流内存失控链，同时消除了 Session Response Body 的 Redis 放大器”； +生产 #1408 的唯一对象级根因仍受限于故障现场没有 heap snapshot。 diff --git a/src/app/v1/_lib/proxy/replay/replay-spool.ts b/src/app/v1/_lib/proxy/replay/replay-spool.ts index 11817432e..4582a06d4 100644 --- a/src/app/v1/_lib/proxy/replay/replay-spool.ts +++ b/src/app/v1/_lib/proxy/replay/replay-spool.ts @@ -27,10 +27,25 @@ import { const FLUSH_INTERVAL_MS = 100; const FLUSH_BYTES_THRESHOLD = 64 * 1024; +const MAX_QUEUED_WRITE_BYTES = 1024 * 1024; const OWNER_HEARTBEAT_INTERVAL_MS = 15_000; const PRE_SPOOL_ABORT_WAIT_MS = 100; let activeSpoolCount = 0; +let durablePersistenceChain: Promise = Promise.resolve(); + +function serializeDurablePersistence(operation: () => Promise): Promise { + const result = durablePersistenceChain.then(operation); + durablePersistenceChain = result.then( + () => undefined, + () => undefined + ); + return result; +} + +export interface ReplaySpoolOptions { + onInactive?: () => void; +} export function getActiveReplaySpoolCount(): number { return activeSpoolCount; @@ -39,10 +54,10 @@ export function getActiveReplaySpoolCount(): number { export class ReplaySpool { private readonly store = getReplayStore(); private readonly decoder = new TextDecoder("utf-8"); - private readonly parts: string[] = []; private readonly queuedBatches = new Set(); private pending: string[] = []; private pendingBytes = 0; + private queuedWriteBytes = 0; private totalBytes = 0; private chunkCount = 0; private disabled = false; @@ -58,7 +73,8 @@ export class ReplaySpool { private readonly ownerToken: string, private readonly statusCode: number, private readonly headers: Record, - private readonly delivery: ReplayDelivery = "stream" + private readonly delivery: ReplayDelivery = "stream", + private readonly options: ReplaySpoolOptions = {} ) { activeSpoolCount++; this.startOwnerHeartbeat(); @@ -79,11 +95,10 @@ export class ReplaySpool { this.disable("payload_too_large"); return; } + this.pendingBytes += chunk.byteLength; const text = this.decoder.decode(chunk, { stream: true }); if (text.length === 0) return; this.pending.push(text); - this.parts.push(text); - this.pendingBytes += chunk.byteLength; if (this.pendingBytes >= FLUSH_BYTES_THRESHOLD) { this.scheduleFlush(0); @@ -119,9 +134,10 @@ export class ReplaySpool { private enqueueFlush(): void { const batch = this.pending; if (batch.length === 0) return; + const batchBytes = this.pendingBytes; this.pending = []; this.pendingBytes = 0; - this.queuedBatches.add(batch); + if (!this.reserveQueuedBatch(batch, batchBytes)) return; // 续接体自带 try/catch：链永不 rejected；每个 await 之后复查 disabled， // 防止与 disable/halt 竞态时在清理之后又写回 owning meta this.writeChain = this.writeChain.then(async () => { @@ -154,10 +170,22 @@ export class ReplaySpool { this.disable("flush_error"); } finally { this.queuedBatches.delete(batch); + this.queuedWriteBytes = Math.max(0, this.queuedWriteBytes - batchBytes); } }); } + private reserveQueuedBatch(batch: string[], batchBytes: number): boolean { + if (this.queuedWriteBytes + batchBytes > MAX_QUEUED_WRITE_BYTES) { + batch.length = 0; + this.disable("write_backlog_too_large"); + return false; + } + this.queuedWriteBytes += batchBytes; + this.queuedBatches.add(batch); + return true; + } + private buildMeta(status: ReplayMeta["status"], extra?: Partial): ReplayMeta { return { status, @@ -236,12 +264,15 @@ export class ReplaySpool { const tail = this.decoder.decode(); if (tail.length > 0) { this.pending.push(tail); - this.parts.push(tail); } const batch = this.pending; + const batchBytes = this.pendingBytes; this.pending = []; this.pendingBytes = 0; - this.queuedBatches.add(batch); + if (!this.reserveQueuedBatch(batch, batchBytes)) { + await this.writeChain; + return; + } this.writeChain = this.writeChain.then(async () => { let pgPersisted = false; @@ -263,21 +294,27 @@ export class ReplaySpool { } this.chunkCount = appended; this.metaWritten = true; - const payload = this.takePayload(); - // 先写 PG（持久 payload），再翻 Redis meta 为 completed（热层可服务） - const persistResult = await this.store.persistCompleted({ - replayId: this.identity.replayId, - verifier: this.identity.verifier, - scopeTag: this.identity.scopeTag, - keyId: this.identity.keyId, - userId: this.identity.userId, - format: this.identity.format, - model: this.identity.model, - statusCode: this.statusCode, - headers: this.headers, - payload, - byteSize: this.totalBytes, - sourceMessageRequestId: messageRequestId, + // Redis 是活跃正文的唯一长期副本。大 payload 的读取、拼接和 PG 写入串行执行， + // 避免多个流同秒终态时在 V8 heap 中并发重建完整响应。 + const persistResult = await serializeDurablePersistence(async () => { + const chunks = await this.store.readChunks(this.identity.replayId, 0); + if (!chunks || chunks.length !== this.chunkCount) { + throw new Error("replay chunks unavailable before durable persistence"); + } + return this.store.persistCompleted({ + replayId: this.identity.replayId, + verifier: this.identity.verifier, + scopeTag: this.identity.scopeTag, + keyId: this.identity.keyId, + userId: this.identity.userId, + format: this.identity.format, + model: this.identity.model, + statusCode: this.statusCode, + headers: this.headers, + payload: chunks.join(""), + byteSize: this.totalBytes, + sourceMessageRequestId: messageRequestId, + }); }); pgPersisted = true; const completed = await this.store.completeOwned( @@ -322,7 +359,7 @@ export class ReplaySpool { .catch(() => false); } finally { this.queuedBatches.delete(batch); - this.clearPayload(); + this.queuedWriteBytes = Math.max(0, this.queuedWriteBytes - batchBytes); this.release(); } }); @@ -338,10 +375,10 @@ export class ReplaySpool { if (this.terminal) return; this.terminal = true; this.aborting = true; + this.notifyInactive(); this.clearTimer(); this.pending = []; this.pendingBytes = 0; - this.clearPayload(); this.clearQueuedBatches(); this.abortPromise = this.writeChain.then(async () => { try { @@ -375,9 +412,9 @@ export class ReplaySpool { private teardown(reason: string, deleteEntry: boolean): void { if (this.disabled) return; this.disabled = true; + this.notifyInactive(); this.clearTimer(); this.pending = []; - this.parts.length = 0; this.pendingBytes = 0; this.clearQueuedBatches(); // 清理顺着 writeChain 串行：与 in-flight append 竞态时绝不出现「删除后又写回」 @@ -435,19 +472,24 @@ export class ReplaySpool { this.clearOwnerHeartbeat(); } - private takePayload(): string { - const payload = this.parts.join(""); - this.clearPayload(); - return payload; - } - - private clearPayload(): void { - this.parts.length = 0; - } - private clearQueuedBatches(): void { for (const batch of this.queuedBatches) batch.length = 0; this.queuedBatches.clear(); + this.queuedWriteBytes = 0; + } + + private inactiveNotified = false; + + private notifyInactive(): void { + if (this.inactiveNotified) return; + this.inactiveNotified = true; + try { + this.options.onInactive?.(); + } catch (error) { + logger.debug("[ReplaySpool] inactive callback failed", { + error: error instanceof Error ? error.message : String(error), + }); + } } } @@ -515,7 +557,8 @@ export function releaseReplayOwnership(session: ProxySession): void { export function createReplaySpoolIfOwner( session: ProxySession, response: Response, - delivery: ReplayDelivery = "stream" + delivery: ReplayDelivery = "stream", + options: ReplaySpoolOptions = {} ): ReplaySpool | null { const replayState = session.replayState; if (replayState?.role !== "owner") return null; @@ -550,7 +593,8 @@ export function createReplaySpoolIfOwner( replayState.ownerToken, response.status, headers, - delivery + delivery, + options ); spool.bootstrap(); return spool; diff --git a/src/app/v1/_lib/proxy/response-handler.ts b/src/app/v1/_lib/proxy/response-handler.ts index 81e78c2cf..de8ad5d62 100644 --- a/src/app/v1/_lib/proxy/response-handler.ts +++ b/src/app/v1/_lib/proxy/response-handler.ts @@ -4147,6 +4147,7 @@ export class ProxyResponseHandler { // 提升 idleTimeoutId 到外部作用域，以便客户端断开时能清除 let idleTimeoutId: NodeJS.Timeout | null = null; let clientAbortDrainTimeoutId: NodeJS.Timeout | null = null; + let clientAbortDrainStartedAt: number | null = null; const streamTextAccumulator = new BoundedStreamTextAccumulator(); let lastStreamTextSnapshot: BoundedStreamTextSnapshot | null = null; const getCollectedChunkCount = () => @@ -4157,6 +4158,45 @@ export class ProxyResponseHandler { clientAbortDrainTimeoutId = null; } }; + const expireClientAbortDrain = () => { + clientAbortDrainTimeoutId = null; + clientAbortDrainStartedAt = null; + logger.info("ResponseHandler: Client abort drain window exceeded", { + taskId, + providerId: provider.id, + messageId: messageContext.id, + clientAbortDrainTimeoutMs, + }); + + try { + const sessionWithController = session as typeof session & { + responseController?: AbortController; + }; + sessionWithController.responseController?.abort(new Error("client_abort_drain_timeout")); + } catch (e) { + logger.warn("ResponseHandler: Failed to abort upstream after client drain timeout", { + taskId, + providerId: provider.id, + error: e, + }); + } + + const drainTimeoutError = new Error("client_abort_drain_timeout"); + abortController.abort(drainTimeoutError); + responsePump?.cancelSource(drainTimeoutError); + }; + const scheduleClientAbortDrainTimeout = (delayMs: number) => { + clearClientAbortDrainTimer(); + clientAbortDrainTimeoutId = setTimeout(expireClientAbortDrain, Math.max(0, delayMs)); + clientAbortDrainTimeoutId.unref?.(); + }; + const capInactiveReplayDrainWindow = () => { + if (clientAbortDrainTimeoutMs <= CLIENT_ABORT_DRAIN_MAX_MS) return; + clientAbortDrainTimeoutMs = CLIENT_ABORT_DRAIN_MAX_MS; + if (clientAbortDrainStartedAt === null) return; + const elapsedMs = Date.now() - clientAbortDrainStartedAt; + scheduleClientAbortDrainTimeout(CLIENT_ABORT_DRAIN_MAX_MS - elapsedMs); + }; const clearIdleTimer = () => { if (idleTimeoutId) { clearTimeout(idleTimeoutId); @@ -4237,31 +4277,8 @@ export class ProxyResponseHandler { if (!idleTimeoutId) { startIdleTimer(); } - clientAbortDrainTimeoutId = setTimeout(() => { - logger.info("ResponseHandler: Client abort drain window exceeded", { - taskId, - providerId: provider.id, - messageId: messageContext.id, - clientAbortDrainTimeoutMs, - }); - - try { - const sessionWithController = session as typeof session & { - responseController?: AbortController; - }; - sessionWithController.responseController?.abort(new Error("client_abort_drain_timeout")); - } catch (e) { - logger.warn("ResponseHandler: Failed to abort upstream after client drain timeout", { - taskId, - providerId: provider.id, - error: e, - }); - } - - const drainTimeoutError = new Error("client_abort_drain_timeout"); - abortController.abort(drainTimeoutError); - responsePump?.cancelSource(drainTimeoutError); - }, clientAbortDrainTimeoutMs); + clientAbortDrainStartedAt = Date.now(); + scheduleClientAbortDrainTimeout(clientAbortDrainTimeoutMs); }; // 统计/结算只保留有界的“头 + 尾”文本快照，避免长流式响应把进程堆撑满。 @@ -4823,7 +4840,9 @@ export class ProxyResponseHandler { // F2 owner spool：guard 阶段已抢到 owner 租约的请求，把客户端可见字节 // write-behind 喂入 Redis 热层，供并发/断线的相同请求 attach 跟尾。 - const replaySpool = createReplaySpoolIfOwner(session, response); + const replaySpool = createReplaySpoolIfOwner(session, response, "stream", { + onInactive: capInactiveReplayDrainWindow, + }); if (replaySpool) { try { clientAbortDrainTimeoutMs = getEnvConfig().REPLAY_MAX_DETACHED_MS; diff --git a/src/lib/config/env.schema.ts b/src/lib/config/env.schema.ts index 7193f9549..2cba1650e 100644 --- a/src/lib/config/env.schema.ts +++ b/src/lib/config/env.schema.ts @@ -157,6 +157,13 @@ export const EnvSchema = z.object({ // - 该开关只影响“写入 Redis 的响应体内容”，不影响内部统计逻辑读取响应体（例如 tokens/费用统计、SSE 结束后的假 200 检测）。 // - message 内容是否脱敏仍由 STORE_SESSION_MESSAGES 控制。 STORE_SESSION_RESPONSE_BODY: z.string().default("true").transform(booleanTransform), + // 单份会话响应正文写入 Redis 的字节上限；旧 response 与 before/after snapshot 都受此边界约束。 + SESSION_RESPONSE_BODY_MAX_BYTES: z.coerce + .number() + .int() + .min(64 * 1024) + .max(64 * 1024 * 1024) + .default(5 * 1024 * 1024), DEBUG_MODE: z.string().default("false").transform(booleanTransform), LOG_LEVEL: z.enum(["fatal", "error", "warn", "info", "debug", "trace"]).default("info"), TZ: z.string().default("Asia/Shanghai"), diff --git a/src/lib/session-manager-detail-snapshots.test.ts b/src/lib/session-manager-detail-snapshots.test.ts index a35ae9cd1..323e847f7 100644 --- a/src/lib/session-manager-detail-snapshots.test.ts +++ b/src/lib/session-manager-detail-snapshots.test.ts @@ -45,6 +45,7 @@ const redisMock = { return Promise.resolve("OK"); }), get: vi.fn((key: string) => Promise.resolve(redisStore.get(key) ?? null)), + del: vi.fn((key: string) => Promise.resolve(redisStore.delete(key) ? 1 : 0)), set: vi.fn().mockResolvedValue("OK"), expire: vi.fn().mockResolvedValue(1), incr: vi.fn().mockResolvedValue(1), @@ -58,11 +59,13 @@ vi.mock("@/lib/redis", () => ({ let mockStoreMessages = false; let mockStoreSessionResponseBody = true; +let mockSessionResponseBodyMaxBytes = 1024 * 1024; vi.mock("@/lib/config/env.schema", () => ({ getEnvConfig: () => ({ STORE_SESSION_MESSAGES: mockStoreMessages, STORE_SESSION_RESPONSE_BODY: mockStoreSessionResponseBody, + SESSION_RESPONSE_BODY_MAX_BYTES: mockSessionResponseBodyMaxBytes, SESSION_TTL: 300, }), })); @@ -76,6 +79,7 @@ describe("SessionManager detail snapshots", () => { redisMock.status = "ready"; mockStoreMessages = false; mockStoreSessionResponseBody = true; + mockSessionResponseBodyMaxBytes = 1024 * 1024; }); it("atomically persists the request sequence while expiring its owner marker", async () => { @@ -392,6 +396,65 @@ describe("SessionManager detail snapshots", () => { }); }); + it("skips only an oversized response body while preserving snapshot headers and meta", async () => { + mockSessionResponseBodyMaxBytes = 4; + + await SessionManager.storeSessionResponsePhaseSnapshot( + "sess_oversized_response", + "after", + { + body: "12345", + headers: new Headers({ "content-type": "text/event-stream" }), + meta: { upstreamUrl: null, statusCode: 200 }, + }, + 1 + ); + + expect( + await SessionManager.getSessionResponsePhaseSnapshot("sess_oversized_response", "after", 1) + ).toEqual({ + body: null, + headers: { "content-type": "text/event-stream" }, + meta: { upstreamUrl: null, statusCode: 200 }, + }); + expect(loggerMock.warn).toHaveBeenCalledWith( + "SessionManager: Skipped oversized session response body", + { context: "snapshot:after", byteSize: 5, maxBytes: 4 } + ); + }); + + it("removes a previous snapshot body when its replacement exceeds the limit", async () => { + mockSessionResponseBodyMaxBytes = 4; + + await SessionManager.storeSessionResponsePhaseSnapshot( + "sess_replaced_response", + "after", + { body: "1234" }, + 1 + ); + await SessionManager.storeSessionResponsePhaseSnapshot( + "sess_replaced_response", + "after", + { + body: "12345", + headers: new Headers({ "content-type": "text/event-stream" }), + meta: { upstreamUrl: null, statusCode: 200 }, + }, + 1 + ); + + expect( + await SessionManager.getSessionResponsePhaseSnapshot("sess_replaced_response", "after", 1) + ).toEqual({ + body: null, + headers: { "content-type": "text/event-stream" }, + meta: { upstreamUrl: null, statusCode: 200 }, + }); + expect(redisMock.del).toHaveBeenCalledWith( + "session:sess_replaced_response:req:1:snapshot:response:after:body" + ); + }); + it("treats empty headers as missing instead of an empty record", async () => { await SessionManager.storeSessionRequestPhaseSnapshot( "sess_empty_headers", diff --git a/src/lib/session-manager.ts b/src/lib/session-manager.ts index 4128f8e90..6b5983420 100644 --- a/src/lib/session-manager.ts +++ b/src/lib/session-manager.ts @@ -1,5 +1,6 @@ import "server-only"; +import { Buffer } from "node:buffer"; import crypto from "node:crypto"; import { extractCodexSessionId } from "@/app/v1/_lib/codex/session-extractor"; import { sanitizeHeaders, sanitizeUrl } from "@/app/v1/_lib/proxy/errors"; @@ -58,6 +59,24 @@ import { SessionTracker } from "./session-tracker"; const RESERVED_INTERNAL_HEADER_SET = new Set( RESERVED_INTERNAL_HEADERS.map((header) => header.toLowerCase()) ); +const DEFAULT_SESSION_RESPONSE_BODY_MAX_BYTES = 5 * 1024 * 1024; + +function canStoreSessionResponseBody(value: string, context: string): boolean { + const configuredMaxBytes = getEnvConfig().SESSION_RESPONSE_BODY_MAX_BYTES; + const maxBytes = + Number.isSafeInteger(configuredMaxBytes) && configuredMaxBytes > 0 + ? configuredMaxBytes + : DEFAULT_SESSION_RESPONSE_BODY_MAX_BYTES; + const byteSize = Buffer.byteLength(value, "utf8"); + if (byteSize <= maxBytes) return true; + + logger.warn("SessionManager: Skipped oversized session response body", { + context, + byteSize, + maxBytes, + }); + return false; +} function isReservedInternalHeader(name: string): boolean { const lowerName = name.toLowerCase(); @@ -2036,7 +2055,7 @@ export class SessionManager { * 存储 session 响应体（临时存储，5分钟过期） * * 存储行为受 STORE_SESSION_RESPONSE_BODY 控制： - * - true (默认)：存储响应体到 Redis 临时缓存 + * - true (默认)：在 SESSION_RESPONSE_BODY_MAX_BYTES 上限内存储响应体到 Redis 临时缓存 * - false：不存储（注意：不影响本次请求处理与统计，仅影响后续查看 response body） * * 存储策略（脱敏/原样）受 STORE_SESSION_MESSAGES 控制： @@ -2061,6 +2080,17 @@ export class SessionManager { if (redis?.status !== "ready") return; try { + // 新格式：session:{sessionId}:req:{sequence}:response（独立存储每个请求） + // 旧格式：session:{sessionId}:response（向后兼容） + const sequence = normalizeRequestSequence(requestSequence); + const key = sequence + ? `session:${sessionId}:req:${sequence}:response` + : `session:${sessionId}:response`; + if (typeof response === "string" && !canStoreSessionResponseBody(response, "response")) { + await redis.del(key); + return; + } + let responseString: string; if (SessionManager.STORE_MESSAGES) { @@ -2082,12 +2112,11 @@ export class SessionManager { } } - // 新格式：session:{sessionId}:req:{sequence}:response（独立存储每个请求） - // 旧格式：session:{sessionId}:response（向后兼容） - const sequence = normalizeRequestSequence(requestSequence); - const key = sequence - ? `session:${sessionId}:req:${sequence}:response` - : `session:${sessionId}:response`; + if (!canStoreSessionResponseBody(responseString, "response")) { + await redis.del(key); + return; + } + if (sequence) { await SessionManager.refreshSessionRequestOwner(redis, sessionId, sequence, keyId); } @@ -2670,8 +2699,24 @@ export class SessionManager { // 与旧平铺 response 字段保持同一隐私/存储契约：关闭时跳过任何 response body phase 落盘。 } else { let bodyToStore = snapshot.body ?? null; + let bodyExceededLimit = false; + const bodyKey = buildSessionDetailSnapshotKey( + sessionId, + sequence, + "response", + phase, + "body" + ); + + if ( + typeof bodyToStore === "string" && + !canStoreSessionResponseBody(bodyToStore, `snapshot:${phase}`) + ) { + bodyToStore = null; + bodyExceededLimit = true; + } - if (!SessionManager.STORE_MESSAGES) { + if (bodyToStore !== null && !SessionManager.STORE_MESSAGES) { if (typeof bodyToStore === "string") { try { bodyToStore = JSON.stringify( @@ -2687,14 +2732,18 @@ export class SessionManager { bodyToStore = JSON.stringify(bodyToStore); } - if (bodyToStore !== null) { - writes.push( - redis.setex( - buildSessionDetailSnapshotKey(sessionId, sequence, "response", phase, "body"), - SessionManager.SESSION_TTL, - bodyToStore - ) - ); + if ( + bodyToStore !== null && + canStoreSessionResponseBody(bodyToStore, `snapshot:${phase}`) + ) { + writes.push(redis.setex(bodyKey, SessionManager.SESSION_TTL, bodyToStore)); + } else if (bodyToStore !== null) { + bodyExceededLimit = true; + } + + if (bodyExceededLimit) { + // 同一 request/phase 可能被重写；超限时删除旧正文，避免读取到上一版小响应。 + writes.push(redis.del(bodyKey)); } } } diff --git a/tests/load/issue-1408-replay-oom/README.md b/tests/load/issue-1408-replay-oom/README.md new file mode 100644 index 000000000..1e794fb45 --- /dev/null +++ b/tests/load/issue-1408-replay-oom/README.md @@ -0,0 +1,96 @@ +# Issue 1408 Replay OOM Load Fixture + +This fixture reproduces the client-disconnect workload used to isolate issue #1408. It keeps the +mock upstream response open after sending a bounded SSE payload, disconnects each client only after +the mock confirms receipt, and samples Node, socket, task, timeout, and Redis state. + +The fixture is intentionally separate from the regular Vitest suite because it needs a configured +CC Hub instance, PostgreSQL, Redis, a Provider pointing at the mock, an API key, and Docker metrics. + +## Files + +- `mock-upstream.cjs`: sends `response.output_text.delta` frames, then remains open without a + terminal event. `CCH_MOCK_MIB` controls the payload size from 0.0625 to 64 MiB per request. +- `drive-disconnect-waves.cjs`: sends distinct `/v1/responses` Replay requests in waves, waits for + the mock receipt count, then disconnects the clients after `CCH_ABORT_DELAY_MS`. +- `memory-probe.cjs`: preload hook that emits RSS, V8 heap, external, ArrayBuffer, and active resource + counts as JSON. +- `sample-container.sh`: samples app logs and optional Redis state into a result file. +- `run-wave.sh`: runs the driver and sampler together. +- `start-mock-container.sh`: starts the mock on an existing Docker network without replacing an + existing container. + +## Prerequisites + +1. Build or select the CC Hub image/revision under test. +2. Start PostgreSQL and create a test database containing a Provider and API key for the fixture. +3. Create a Docker network shared by the app, Redis, and mock. +4. Configure the Provider base URL as `http://MOCK_CONTAINER:3001` and route model `gpt-5.6` to it. +5. Start the app with the probe preloaded. For a container, mount `memory-probe.cjs` read-only and + set `NODE_OPTIONS=--require=/fixture/memory-probe.cjs`. + +Do not point this fixture at a production Provider. The mock deliberately leaves every upstream +response open until the app or fixture closes it. + +## Start The Mock + +```bash +tests/load/issue-1408-replay-oom/start-mock-container.sh \ + cch1408-mock cch1408-network 31409 8 +``` + +The command prints both URLs: + +```text +stats=http://127.0.0.1:31409/stats +provider=http://cch1408-mock:3001 +``` + +## Run A Wave Test + +Store the test API key in a protected file outside the repository, then run: + +```bash +export CCH_API_KEY_FILE=/path/to/test-api-key + +tests/load/issue-1408-replay-oom/run-wave.sh \ + http://127.0.0.1:31415 \ + http://127.0.0.1:31409/stats \ + issue1408-fixed \ + cch1408-app \ + issue1408-fixed.samples.txt \ + cch1408-redis +``` + +Defaults match the investigation workload: + +```text +CCH_WAVES=8 +CCH_REQUESTS_PER_WAVE=8 +CCH_WAVE_INTERVAL_MS=10000 +CCH_ABORT_DELAY_MS=250 +CCH_SAMPLES=18 +CCH_SAMPLE_INTERVAL_SECONDS=10 +CCH_MOCK_MIB=8 +``` + +Use a unique scenario prefix for every run. Replay identity includes the scenario, wave, and request +index, so a unique prefix prevents a previous durable Replay entry from turning the workload into a +cache hit. Scenario prefixes accept 1 to 64 ASCII letters, digits, underscores, and hyphens. The +driver destroys all requests if mock receipt confirmation fails, so a failed run does not leave its +own upstream streams active. + +## Acceptance Signals + +For the fixed revision under the default workload: + +- Node heap remains bounded while external and ArrayBuffer memory follow active stream count. +- `write_backlog_too_large` makes an inactive Replay spool fall back to the 60-second drain window. +- Every disconnected task reaches `Client abort drain window exceeded` and the active task count + returns to zero. +- After a quiet GC period, external and ArrayBuffer memory return near the pre-wave baseline. +- Redis remains running across its configured RDB save window and does not retain three copies of + each multi-MiB response body. + +The historical measurements and the exact evidence boundary are documented in +`docs/troubleshooting/issue-1408-replay-oom.md`. diff --git a/tests/load/issue-1408-replay-oom/drive-disconnect-waves.cjs b/tests/load/issue-1408-replay-oom/drive-disconnect-waves.cjs new file mode 100644 index 000000000..8fff27546 --- /dev/null +++ b/tests/load/issue-1408-replay-oom/drive-disconnect-waves.cjs @@ -0,0 +1,216 @@ +"use strict"; + +const fs = require("node:fs"); +const http = require("node:http"); +const https = require("node:https"); + +const [appUrl, mockStatsUrl, scenarioPrefix, wavesArg, perWaveArg, intervalArg] = + process.argv.slice(2); + +if (!appUrl || !mockStatsUrl || !scenarioPrefix) { + throw new Error( + "usage: drive-disconnect-waves.cjs APP_URL MOCK_STATS_URL SCENARIO_PREFIX " + + "[WAVES] [REQUESTS_PER_WAVE] [INTERVAL_MS]" + ); +} + +const parsedAppUrl = parseHttpUrl(appUrl, "APP_URL"); +const parsedMockStatsUrl = parseHttpUrl(mockStatsUrl, "MOCK_STATS_URL"); +const normalizedScenarioPrefix = parseScenarioPrefix(scenarioPrefix); +const waves = parseBoundedInteger(wavesArg || "8", "WAVES", 1, 255); +const perWave = parseBoundedInteger(perWaveArg || "8", "REQUESTS_PER_WAVE", 1, 255); +const intervalMs = parseBoundedInteger(intervalArg || "10000", "INTERVAL_MS", 0, 3600000); +const abortDelayMs = parseBoundedInteger( + process.env.CCH_ABORT_DELAY_MS || "250", + "CCH_ABORT_DELAY_MS", + 0, + 60000 +); +const mockReceiptTimeoutMs = parseBoundedInteger( + process.env.CCH_MOCK_RECEIPT_TIMEOUT_MS || "30000", + "CCH_MOCK_RECEIPT_TIMEOUT_MS", + 1, + 3600000 +); +const model = (process.env.CCH_REQUEST_MODEL || "gpt-5.6").trim(); +if (!model || model.length > 256) { + throw new Error("CCH_REQUEST_MODEL must contain between 1 and 256 characters"); +} +const key = readApiKey(); + +function parseHttpUrl(raw, name) { + const url = new URL(raw); + if (url.protocol !== "http:" && url.protocol !== "https:") { + throw new Error(`${name} must use http or https`); + } + return url; +} + +function parseScenarioPrefix(raw) { + if (!/^[a-z0-9_-]{1,64}$/i.test(raw)) { + throw new Error("SCENARIO_PREFIX must match [a-z0-9_-] and contain 1 to 64 characters"); + } + return raw; +} + +function parseBoundedInteger(raw, name, minimum, maximum) { + const value = Number(raw); + if (!Number.isInteger(value) || value < minimum || value > maximum) { + throw new Error(`${name} must be an integer between ${minimum} and ${maximum}`); + } + return value; +} + +function readApiKey() { + const direct = process.env.CCH_API_KEY?.trim(); + if (direct) return direct; + const keyFile = process.env.CCH_API_KEY_FILE; + if (keyFile) { + const value = fs.readFileSync(keyFile, "utf8").trim(); + if (value) return value; + } + throw new Error("set CCH_API_KEY or CCH_API_KEY_FILE before running the fixture"); +} + +function transportFor(url) { + return url.protocol === "https:" ? https : http; +} + +function sleep(delayMs) { + return new Promise((resolve) => setTimeout(resolve, delayMs)); +} + +function getJson(rawUrl) { + return new Promise((resolve, reject) => { + const url = rawUrl instanceof URL ? rawUrl : new URL(rawUrl); + const request = transportFor(url).get(url, (response) => { + const chunks = []; + response.once("aborted", () => reject(new Error(`GET ${url} response aborted`))); + response.once("error", reject); + response.on("data", (chunk) => chunks.push(chunk)); + response.on("end", () => { + if ((response.statusCode || 500) >= 400) { + reject(new Error(`GET ${url} returned ${response.statusCode}`)); + return; + } + try { + resolve(JSON.parse(Buffer.concat(chunks).toString("utf8"))); + } catch (error) { + reject(error); + } + }); + }); + request.setTimeout(mockReceiptTimeoutMs, () => request.destroy(new Error("stats timeout"))); + request.on("error", reject); + }); +} + +async function waitForMock(scenario, target) { + const deadline = Date.now() + mockReceiptTimeoutMs; + while (Date.now() < deadline) { + const stats = await getJson(mockStatsUrl); + if ((stats.counts?.[scenario] || 0) >= target) return stats.counts[scenario]; + await sleep(50); + } + throw new Error(`mock receipt timeout for ${scenario}: target=${target}`); +} + +function hashScenario(value) { + return [...value].reduce((hash, char) => (hash * 33 + char.charCodeAt(0)) & 0xff, 0); +} + +function startRequest(scenario, scenarioHash, wave, index) { + const body = JSON.stringify({ + model, + input: [ + { + role: "user", + content: [ + { + type: "input_text", + text: `CCH_SCENARIO_${scenario} wave-${wave} request-${index}`, + }, + ], + }, + ], + stream: true, + prompt_cache_key: `cch1408-${scenario}-${wave}-${index}`, + }); + const url = new URL("/v1/responses", parsedAppUrl); + const suffix = `${scenarioHash.toString(16).padStart(2, "0")}${(wave + 1) + .toString(16) + .padStart(2, "0")}${index.toString(16).padStart(2, "0")}000000`; + const handle = { request: null, response: null }; + const request = transportFor(url).request( + url, + { + method: "POST", + headers: { + authorization: `Bearer ${key}`, + "content-type": "application/json", + "content-length": Buffer.byteLength(body), + session_id: `019c1408-0000-7000-8000-${suffix}`, + }, + }, + (response) => { + handle.response = response; + response.on("data", () => {}); + response.on("error", () => {}); + } + ); + handle.request = request; + request.on("error", () => {}); + request.end(body); + return handle; +} + +function abortRequests(handles) { + for (const handle of handles) { + handle.response?.destroy(); + handle.request?.destroy(); + } +} + +async function main() { + const scenarioHash = hashScenario(normalizedScenarioPrefix); + for (let wave = 0; wave < waves; wave += 1) { + const scenario = `${normalizedScenarioPrefix}-${wave}`; + const before = (await getJson(parsedMockStatsUrl)).counts?.[scenario] || 0; + const handles = []; + for (let index = 0; index < perWave; index += 1) { + handles.push(startRequest(scenario, scenarioHash, wave, index)); + } + + let received; + let confirmedAt; + let abortedAt; + try { + received = await waitForMock(scenario, before + perWave); + confirmedAt = Date.now(); + await sleep(abortDelayMs); + abortedAt = Date.now(); + } finally { + abortRequests(handles); + } + + process.stdout.write( + `${JSON.stringify({ + wave, + scenario, + perWave, + mockBefore: before, + mockReceived: received, + confirmedAt, + abortedAt, + abortDelayMs: abortedAt - confirmedAt, + })}\n` + ); + + if (wave + 1 < waves) await sleep(intervalMs); + } +} + +main().catch((error) => { + process.stderr.write(`${error instanceof Error ? error.stack : String(error)}\n`); + process.exitCode = 1; +}); diff --git a/tests/load/issue-1408-replay-oom/memory-probe.cjs b/tests/load/issue-1408-replay-oom/memory-probe.cjs new file mode 100644 index 000000000..4a6fa680a --- /dev/null +++ b/tests/load/issue-1408-replay-oom/memory-probe.cjs @@ -0,0 +1,41 @@ +"use strict"; + +const v8 = require("node:v8"); + +const rawInterval = process.env.CCH_MEMORY_PROBE_INTERVAL_MS || "1000"; +const intervalMs = Number(rawInterval); +if (!Number.isInteger(intervalMs) || intervalMs < 10 || intervalMs > 60000) { + throw new Error("CCH_MEMORY_PROBE_INTERVAL_MS must be an integer between 10 and 60000"); +} + +function toMiB(value) { + return Number((value / 1048576).toFixed(2)); +} + +function sample() { + const memory = process.memoryUsage(); + const resources = + typeof process.getActiveResourcesInfo === "function" ? process.getActiveResourcesInfo() : []; + const resourceCounts = {}; + for (const name of resources) { + resourceCounts[name] = (resourceCounts[name] || 0) + 1; + } + + process.stdout.write( + `${JSON.stringify({ + cchMemoryProbe: true, + ts: Date.now(), + rssMiB: toMiB(memory.rss), + heapUsedMiB: toMiB(memory.heapUsed), + heapTotalMiB: toMiB(memory.heapTotal), + externalMiB: toMiB(memory.external), + arrayBuffersMiB: toMiB(memory.arrayBuffers), + mallocedMiB: toMiB(v8.getHeapStatistics().malloced_memory), + resources: resourceCounts, + })}\n` + ); +} + +const timer = setInterval(sample, intervalMs); +timer.unref(); +sample(); diff --git a/tests/load/issue-1408-replay-oom/mock-upstream.cjs b/tests/load/issue-1408-replay-oom/mock-upstream.cjs new file mode 100644 index 000000000..d3cbd6598 --- /dev/null +++ b/tests/load/issue-1408-replay-oom/mock-upstream.cjs @@ -0,0 +1,158 @@ +"use strict"; + +const http = require("node:http"); + +const host = process.env.CCH_MOCK_HOST || "0.0.0.0"; +const port = parseBoundedInteger(process.env.CCH_MOCK_PORT || "3001", "CCH_MOCK_PORT", 0, 65535); +const totalMiB = parseBoundedNumber(process.env.CCH_MOCK_MIB || "8", "CCH_MOCK_MIB", 0.0625, 64); +const maxRequestBytes = parseBoundedInteger( + process.env.CCH_MOCK_MAX_REQUEST_BYTES || String(1024 * 1024), + "CCH_MOCK_MAX_REQUEST_BYTES", + 1, + 16 * 1024 * 1024 +); +const chunkText = "x".repeat(64 * 1024); +const framesPerMiB = 16; +const counts = new Map(); + +function parseBoundedNumber(raw, name, minimum, maximum) { + const value = Number(raw); + if (!Number.isFinite(value) || value < minimum || value > maximum) { + throw new Error(`${name} must be a number between ${minimum} and ${maximum}`); + } + return value; +} + +function parseBoundedInteger(raw, name, minimum, maximum) { + const value = Number(raw); + if (!Number.isInteger(value) || value < minimum || value > maximum) { + throw new Error(`${name} must be an integer between ${minimum} and ${maximum}`); + } + return value; +} + +function readBody(req) { + return new Promise((resolve, reject) => { + const chunks = []; + let receivedBytes = 0; + let tooLarge = false; + req.on("data", (chunk) => { + if (tooLarge) return; + receivedBytes += chunk.byteLength; + if (receivedBytes > maxRequestBytes) { + tooLarge = true; + chunks.length = 0; + reject(new Error("request body too large")); + return; + } + chunks.push(chunk); + }); + req.on("end", () => { + if (!tooLarge) resolve(Buffer.concat(chunks).toString("utf8")); + }); + req.on("error", (error) => { + if (!tooLarge) reject(error); + }); + }); +} + +function scenarioFrom(raw) { + const match = raw.match(/CCH_SCENARIO_([a-z0-9_-]+)/i); + return match ? match[1] : "unknown"; +} + +function writeJson(res, statusCode, value) { + res.writeHead(statusCode, { "content-type": "application/json" }); + res.end(JSON.stringify(value)); +} + +const server = http.createServer(async (req, res) => { + if (req.method === "GET" && (req.url === "/health" || req.url === "/stats")) { + writeJson(res, 200, { + counts: Object.fromEntries(counts), + totalMiB, + }); + return; + } + + if (req.method === "POST" && req.url === "/reset") { + counts.clear(); + writeJson(res, 200, { reset: true }); + return; + } + + if (req.method !== "POST" || req.url !== "/v1/responses") { + writeJson(res, 404, { error: "not found" }); + return; + } + + let raw; + try { + raw = await readBody(req); + } catch (error) { + if (!res.headersSent && !res.destroyed) { + writeJson(res, 413, { error: error instanceof Error ? error.message : String(error) }); + } + return; + } + + const scenario = scenarioFrom(raw); + counts.set(scenario, (counts.get(scenario) || 0) + 1); + + process.stdout.write( + `${JSON.stringify({ + event: "request", + path: req.url, + requestBytes: Buffer.byteLength(raw), + scenario, + ordinal: counts.get(scenario), + totalMiB, + })}\n` + ); + + res.on("error", () => {}); + res.writeHead(200, { + "content-type": "text/event-stream", + "cache-control": "no-cache", + connection: "keep-alive", + }); + + const totalFrames = Math.max(1, Math.ceil(totalMiB * framesPerMiB)); + let sent = 0; + const writeNext = () => { + if (res.destroyed || sent >= totalFrames) return; + sent += 1; + const event = `data: ${JSON.stringify({ + type: "response.output_text.delta", + delta: chunkText, + })}\n\n`; + if (!res.write(event)) { + res.once("drain", writeNext); + } else { + setImmediate(writeNext); + } + }; + writeNext(); +}); + +const sockets = new Set(); +server.on("connection", (socket) => { + sockets.add(socket); + socket.on("close", () => sockets.delete(socket)); +}); + +function shutdown() { + server.close(() => process.exit(0)); + for (const socket of sockets) socket.destroy(); +} + +process.on("SIGINT", shutdown); +process.on("SIGTERM", shutdown); + +server.listen(port, host, () => { + const address = server.address(); + const listeningPort = typeof address === "object" && address ? address.port : port; + process.stdout.write( + `${JSON.stringify({ event: "listening", host, port: listeningPort, totalMiB })}\n` + ); +}); diff --git a/tests/load/issue-1408-replay-oom/run-wave.sh b/tests/load/issue-1408-replay-oom/run-wave.sh new file mode 100755 index 000000000..564fdda6e --- /dev/null +++ b/tests/load/issue-1408-replay-oom/run-wave.sh @@ -0,0 +1,51 @@ +#!/bin/sh +set -eu + +if [ "$#" -lt 5 ] || [ "$#" -gt 6 ]; then + printf '%s\n' \ + "usage: run-wave.sh APP_URL MOCK_STATS_URL SCENARIO_PREFIX APP_CONTAINER OUTPUT [REDIS_CONTAINER]" >&2 + exit 2 +fi + +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +app_url="$1" +mock_stats_url="$2" +scenario_prefix="$3" +app_container="$4" +output="$5" +redis_container="${6:-}" + +waves="${CCH_WAVES:-8}" +requests_per_wave="${CCH_REQUESTS_PER_WAVE:-8}" +wave_interval_ms="${CCH_WAVE_INTERVAL_MS:-10000}" +samples="${CCH_SAMPLES:-18}" +sample_interval_seconds="${CCH_SAMPLE_INTERVAL_SECONDS:-10}" +node_bin="${NODE_BIN:-node}" + +sampler_pid="" +cleanup() { + if [ -n "$sampler_pid" ]; then + kill "$sampler_pid" 2>/dev/null || true + fi +} +trap cleanup EXIT INT TERM + +"$script_dir/sample-container.sh" \ + "$app_container" \ + "$output" \ + "$samples" \ + "$sample_interval_seconds" \ + "$redis_container" & +sampler_pid=$! + +"$node_bin" "$script_dir/drive-disconnect-waves.cjs" \ + "$app_url" \ + "$mock_stats_url" \ + "$scenario_prefix" \ + "$waves" \ + "$requests_per_wave" \ + "$wave_interval_ms" + +wait "$sampler_pid" +sampler_pid="" +trap - EXIT INT TERM diff --git a/tests/load/issue-1408-replay-oom/sample-container.sh b/tests/load/issue-1408-replay-oom/sample-container.sh new file mode 100755 index 000000000..ba202e560 --- /dev/null +++ b/tests/load/issue-1408-replay-oom/sample-container.sh @@ -0,0 +1,50 @@ +#!/bin/sh +set -eu + +if [ "$#" -lt 2 ] || [ "$#" -gt 5 ]; then + printf '%s\n' \ + "usage: sample-container.sh APP_CONTAINER OUTPUT [SAMPLES] [INTERVAL_SECONDS] [REDIS_CONTAINER]" >&2 + exit 2 +fi + +app="$1" +output="$2" +samples="${3:-18}" +interval="${4:-10}" +redis="${5:-}" + +case "$samples" in + *[!0-9]* | 0) printf '%s\n' "SAMPLES must be a positive integer" >&2; exit 2 ;; +esac +case "$interval" in + *[!0-9]*) printf '%s\n' "INTERVAL_SECONDS must be a non-negative integer" >&2; exit 2 ;; +esac + +start_epoch=$(date +%s) +: >"$output" +i=0 +while [ "$i" -lt "$samples" ]; do + epoch=$(date +%s) + state=$(docker inspect -f '{{.State.Status}} oom={{.State.OOMKilled}} exit={{.State.ExitCode}}' "$app" 2>/dev/null || true) + logs=$(docker logs --since "$start_epoch" "$app" 2>&1 || true) + memory=$(printf '%s\n' "$logs" | grep '"cchMemoryProbe":true' | tail -n 1 || true) + timeouts=$(printf '%s\n' "$logs" | grep -c 'Client abort drain window exceeded' || true) + backlogs=$(printf '%s\n' "$logs" | grep -c 'write_backlog_too_large' || true) + body_skips=$(printf '%s\n' "$logs" | grep -c 'Skipped oversized session response body' || true) + active=$(printf '%s\n' "$logs" | grep -E 'activeTasks|remainingTasks' | tail -n 1 || true) + + redis_state="" + redis_memory="" + if [ -n "$redis" ]; then + redis_state=$(docker inspect -f '{{.State.Status}} oom={{.State.OOMKilled}} exit={{.State.ExitCode}}' "$redis" 2>/dev/null || true) + redis_memory=$(docker exec "$redis" redis-cli --raw INFO memory 2>/dev/null | + grep -E '^(used_memory_human|used_memory_peak_human):' | + tr '\n' ',' || true) + fi + + printf '%s\n' \ + "sample=$i epoch=$epoch app=[$state] timeouts=$timeouts backlogs=$backlogs bodySkips=$body_skips memory=$memory lastTask=$active redis=[$redis_state] redisMemory=[$redis_memory]" \ + >>"$output" + i=$((i + 1)) + [ "$i" -ge "$samples" ] || sleep "$interval" +done diff --git a/tests/load/issue-1408-replay-oom/start-mock-container.sh b/tests/load/issue-1408-replay-oom/start-mock-container.sh new file mode 100755 index 000000000..3c9c0869e --- /dev/null +++ b/tests/load/issue-1408-replay-oom/start-mock-container.sh @@ -0,0 +1,54 @@ +#!/bin/sh +set -eu + +if [ "$#" -lt 3 ] || [ "$#" -gt 5 ]; then + printf '%s\n' \ + "usage: start-mock-container.sh CONTAINER NETWORK HOST_PORT [PAYLOAD_MIB] [NODE_IMAGE]" >&2 + exit 2 +fi + +container="$1" +network="$2" +host_port="$3" +payload_mib="${4:-8}" +node_image="${5:-node:22-alpine}" +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) + +if docker container inspect "$container" >/dev/null 2>&1; then + printf '%s\n' "container already exists: $container" >&2 + exit 1 +fi + +docker run -d \ + --name "$container" \ + --network "$network" \ + -e CCH_MOCK_PORT=3001 \ + -e CCH_MOCK_MIB="$payload_mib" \ + -p "127.0.0.1:$host_port:3001" \ + -v "$script_dir/mock-upstream.cjs:/fixture/mock-upstream.cjs:ro" \ + "$node_image" \ + node /fixture/mock-upstream.cjs >/dev/null + +cleanup_container() { + docker rm -f "$container" >/dev/null 2>&1 || true +} + +trap cleanup_container 0 +trap 'exit 130' 2 +trap 'exit 143' 15 + +attempt=0 +while [ "$attempt" -lt 60 ]; do + if curl --connect-timeout 2 --max-time 5 -fsS \ + "http://127.0.0.1:$host_port/health" >/dev/null 2>&1; then + trap - 0 2 15 + printf '%s\n' \ + "ready container=$container stats=http://127.0.0.1:$host_port/stats provider=http://$container:3001" + exit 0 + fi + attempt=$((attempt + 1)) + sleep 1 +done + +docker logs --tail 100 "$container" >&2 || true +exit 1 diff --git a/tests/unit/lib/env-store-session-response-body.test.ts b/tests/unit/lib/env-store-session-response-body.test.ts index bcb25c45c..da16baf13 100644 --- a/tests/unit/lib/env-store-session-response-body.test.ts +++ b/tests/unit/lib/env-store-session-response-body.test.ts @@ -3,6 +3,7 @@ import { EnvSchema } from "@/lib/config/env.schema"; describe("EnvSchema - STORE_SESSION_RESPONSE_BODY", () => { const originalEnv = process.env.STORE_SESSION_RESPONSE_BODY; + const originalMaxBytes = process.env.SESSION_RESPONSE_BODY_MAX_BYTES; afterEach(() => { if (originalEnv === undefined) { @@ -10,6 +11,11 @@ describe("EnvSchema - STORE_SESSION_RESPONSE_BODY", () => { } else { process.env.STORE_SESSION_RESPONSE_BODY = originalEnv; } + if (originalMaxBytes === undefined) { + delete process.env.SESSION_RESPONSE_BODY_MAX_BYTES; + } else { + process.env.SESSION_RESPONSE_BODY_MAX_BYTES = originalMaxBytes; + } }); it("should default to true when not set", () => { @@ -41,4 +47,26 @@ describe("EnvSchema - STORE_SESSION_RESPONSE_BODY", () => { const result = EnvSchema.parse(process.env); expect(result.STORE_SESSION_RESPONSE_BODY).toBe(true); }); + + it("defaults the response body limit to 5 MiB", () => { + delete process.env.SESSION_RESPONSE_BODY_MAX_BYTES; + const result = EnvSchema.parse(process.env); + expect(result.SESSION_RESPONSE_BODY_MAX_BYTES).toBe(5 * 1024 * 1024); + }); + + it("accepts the inclusive 64 KiB and 64 MiB response body limit boundaries", () => { + process.env.SESSION_RESPONSE_BODY_MAX_BYTES = String(64 * 1024); + expect(EnvSchema.parse(process.env).SESSION_RESPONSE_BODY_MAX_BYTES).toBe(64 * 1024); + + process.env.SESSION_RESPONSE_BODY_MAX_BYTES = String(64 * 1024 * 1024); + expect(EnvSchema.parse(process.env).SESSION_RESPONSE_BODY_MAX_BYTES).toBe(64 * 1024 * 1024); + }); + + it("rejects response body limits outside the configured boundaries", () => { + process.env.SESSION_RESPONSE_BODY_MAX_BYTES = String(64 * 1024 - 1); + expect(() => EnvSchema.parse(process.env)).toThrow(); + + process.env.SESSION_RESPONSE_BODY_MAX_BYTES = String(64 * 1024 * 1024 + 1); + expect(() => EnvSchema.parse(process.env)).toThrow(); + }); }); diff --git a/tests/unit/lib/session-manager-redaction.test.ts b/tests/unit/lib/session-manager-redaction.test.ts index 3cc3c24f3..e3fb9ffb3 100644 --- a/tests/unit/lib/session-manager-redaction.test.ts +++ b/tests/unit/lib/session-manager-redaction.test.ts @@ -31,6 +31,7 @@ const redisMock = { status: "ready", setex: vi.fn().mockResolvedValue("OK"), get: vi.fn(), + del: vi.fn().mockResolvedValue(1), set: vi.fn().mockResolvedValue("OK"), expire: vi.fn().mockResolvedValue(1), incr: vi.fn().mockResolvedValue(1), @@ -49,10 +50,12 @@ vi.mock("@/lib/redis", () => ({ // Mock config - we'll control STORE_SESSION_MESSAGES dynamically let mockStoreMessages = false; let mockStoreSessionResponseBody = true; +let mockSessionResponseBodyMaxBytes = 1024 * 1024; vi.mock("@/lib/config/env.schema", () => ({ getEnvConfig: () => ({ STORE_SESSION_MESSAGES: mockStoreMessages, STORE_SESSION_RESPONSE_BODY: mockStoreSessionResponseBody, + SESSION_RESPONSE_BODY_MAX_BYTES: mockSessionResponseBodyMaxBytes, SESSION_TTL: 300, }), })); @@ -65,11 +68,13 @@ describe("SessionManager - Redaction based on STORE_SESSION_MESSAGES", () => { vi.clearAllMocks(); mockStoreMessages = false; // default: redact mockStoreSessionResponseBody = true; // default: store response body + mockSessionResponseBodyMaxBytes = 1024 * 1024; }); afterEach(() => { mockStoreMessages = false; mockStoreSessionResponseBody = true; + mockSessionResponseBodyMaxBytes = 1024 * 1024; }); describe("storeSessionMessages", () => { @@ -215,6 +220,37 @@ describe("SessionManager - Redaction based on STORE_SESSION_MESSAGES", () => { expect(value).toBe(nonJsonResponse); }); + it("should enforce the response body limit using UTF-8 bytes at the exact boundary", async () => { + mockSessionResponseBodyMaxBytes = 4; + + await SessionManager.storeSessionResponse("sess_utf8_exact", "中a", 1); + expect(redisMock.setex).toHaveBeenCalledWith( + "session:sess_utf8_exact:req:1:response", + 300, + "中a" + ); + + vi.clearAllMocks(); + await SessionManager.storeSessionResponse("sess_utf8_over", "中ab", 1); + + expect(redisMock.setex).not.toHaveBeenCalled(); + expect(loggerMock.warn).toHaveBeenCalledWith( + "SessionManager: Skipped oversized session response body", + { context: "response", byteSize: 5, maxBytes: 4 } + ); + expect(redisMock.del).toHaveBeenCalledWith("session:sess_utf8_over:req:1:response"); + }); + + it("should remove a previously stored response when the replacement exceeds the limit", async () => { + mockSessionResponseBodyMaxBytes = 4; + + await SessionManager.storeSessionResponse("sess_replace", "1234", 1); + await SessionManager.storeSessionResponse("sess_replace", "12345", 1); + + expect(redisMock.setex).toHaveBeenCalledTimes(1); + expect(redisMock.del).toHaveBeenCalledWith("session:sess_replace:req:1:response"); + }); + it("should handle OpenAI choices format when STORE_SESSION_MESSAGES=false", async () => { mockStoreMessages = false; const openaiResponse = { diff --git a/tests/unit/proxy/issue-1408-load-fixture.test.ts b/tests/unit/proxy/issue-1408-load-fixture.test.ts new file mode 100644 index 000000000..55a8478fd --- /dev/null +++ b/tests/unit/proxy/issue-1408-load-fixture.test.ts @@ -0,0 +1,520 @@ +import { + execFileSync, + spawn, + spawnSync, + type ChildProcessWithoutNullStreams, +} from "node:child_process"; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import http from "node:http"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; + +const fixtureDir = path.join(process.cwd(), "tests/load/issue-1408-replay-oom"); +const nodeScripts = ["mock-upstream.cjs", "drive-disconnect-waves.cjs", "memory-probe.cjs"]; +const shellScripts = ["sample-container.sh", "run-wave.sh", "start-mock-container.sh"]; +const children = new Set(); +const posixIt = it.skipIf(process.platform === "win32"); +const mockEnvironmentKeys = [ + "CCH_MOCK_HOST", + "CCH_MOCK_PORT", + "CCH_MOCK_MIB", + "CCH_MOCK_MAX_REQUEST_BYTES", +] as const; + +function createMockEnvironment(overrides: Record = {}): NodeJS.ProcessEnv { + const environment = { ...process.env }; + for (const key of mockEnvironmentKeys) delete environment[key]; + return { + ...environment, + CCH_MOCK_HOST: "127.0.0.1", + CCH_MOCK_PORT: "0", + CCH_MOCK_MIB: "0.0625", + ...overrides, + }; +} + +function getJson(url: URL): Promise> { + return new Promise((resolve, reject) => { + const request = http.get(url, (response) => { + const chunks: Buffer[] = []; + response.on("data", (chunk: Buffer) => chunks.push(chunk)); + response.on("end", () => { + try { + resolve(JSON.parse(Buffer.concat(chunks).toString("utf8")) as Record); + } catch (error) { + reject(error); + } + }); + }); + request.on("error", reject); + }); +} + +function requestJson( + url: URL, + method: string, + body = "" +): Promise<{ statusCode: number; body: Record }> { + return new Promise((resolve, reject) => { + const request = http.request( + url, + { + method, + headers: body + ? { + "content-type": "application/json", + "content-length": Buffer.byteLength(body), + } + : undefined, + }, + (response) => { + const chunks: Buffer[] = []; + response.on("data", (chunk: Buffer) => chunks.push(chunk)); + response.on("end", () => { + try { + resolve({ + statusCode: response.statusCode ?? 0, + body: JSON.parse(Buffer.concat(chunks).toString("utf8")) as Record, + }); + } catch (error) { + reject(error); + } + }); + } + ); + request.on("error", reject); + request.end(body); + }); +} + +async function startMock( + overrides: Record = {} +): Promise<{ child: ChildProcessWithoutNullStreams; baseUrl: URL }> { + const child = spawn(process.execPath, [path.join(fixtureDir, "mock-upstream.cjs")], { + env: createMockEnvironment(overrides), + stdio: ["ignore", "pipe", "pipe"], + }); + children.add(child); + + const listening = await waitForJsonLine(child, (value) => value.event === "listening"); + expect(listening.port).toEqual(expect.any(Number)); + return { child, baseUrl: new URL(`http://127.0.0.1:${listening.port}`) }; +} + +function waitForJsonLine( + child: ChildProcessWithoutNullStreams, + predicate: (value: Record) => boolean +): Promise> { + return new Promise((resolve, reject) => { + let buffered = ""; + const timer = setTimeout(() => reject(new Error("fixture output timeout")), 5000); + child.stdout.on("data", (chunk: Buffer) => { + buffered += chunk.toString("utf8"); + const lines = buffered.split("\n"); + buffered = lines.pop() ?? ""; + for (const line of lines) { + if (!line.trim()) continue; + const value = JSON.parse(line) as Record; + if (predicate(value)) { + clearTimeout(timer); + resolve(value); + return; + } + } + }); + child.once("error", (error) => { + clearTimeout(timer); + reject(error); + }); + child.once("exit", (code) => { + clearTimeout(timer); + reject(new Error(`fixture exited before readiness: ${code}`)); + }); + }); +} + +function postAndAbort(url: URL, body: string): Promise { + return new Promise((resolve, reject) => { + const request = http.request( + url, + { + method: "POST", + headers: { + "content-type": "application/json", + "content-length": Buffer.byteLength(body), + }, + }, + (response) => { + response.once("data", () => { + response.destroy(); + resolve(); + }); + response.on("error", () => resolve()); + } + ); + request.on("error", reject); + request.end(body); + }); +} + +function waitForExit( + child: ChildProcessWithoutNullStreams +): Promise<{ code: number | null; stderr: string }> { + return new Promise((resolve, reject) => { + let stderr = ""; + child.stderr.on("data", (chunk: Buffer) => { + stderr += chunk.toString("utf8"); + }); + child.once("error", reject); + child.once("exit", (code) => resolve({ code, stderr })); + }); +} + +function installFakeContainerCommands(directory: string): void { + writeFileSync( + path.join(directory, "docker"), + `#!/bin/sh +set -eu +printf '%s\\n' "$*" >> "$CCH_FAKE_DOCKER_LOG" +case "$1" in + container) + [ "$2" = "inspect" ] || exit 2 + exit "\${CCH_FAKE_CONTAINER_INSPECT_STATUS:-1}" + ;; + inspect) + exit "\${CCH_FAKE_IMAGE_INSPECT_STATUS:-0}" + ;; + run) + : > "$CCH_FAKE_CONTAINER_STATE" + ;; + logs) + ;; + rm) + rm -f "$CCH_FAKE_CONTAINER_STATE" + ;; + *) + exit 2 + ;; +esac +`, + { mode: 0o755 } + ); + writeFileSync( + path.join(directory, "curl"), + `#!/bin/sh +printf '%s\\n' "$*" >> "$CCH_FAKE_CURL_LOG" +exit "\${CCH_FAKE_CURL_STATUS:-1}" +`, + { mode: 0o755 } + ); + writeFileSync( + path.join(directory, "sleep"), + `#!/bin/sh +case "\${CCH_FAKE_SLEEP_MODE:-success}" in + signal-int) + kill -INT "$PPID" + ;; + signal-term) + kill -TERM "$PPID" + ;; + fail) + exit 7 + ;; +esac +`, + { mode: 0o755 } + ); +} + +function runStartMockWithFakeCommands( + directory: string, + overrides: Record +): ReturnType { + return spawnSync( + "sh", + [ + path.join(fixtureDir, "start-mock-container.sh"), + "fixture-container", + "fixture-network", + "31409", + ], + { + encoding: "utf8", + env: { + ...process.env, + PATH: `${directory}:${process.env.PATH ?? ""}`, + CCH_FAKE_CONTAINER_STATE: path.join(directory, "container-state"), + CCH_FAKE_CURL_LOG: path.join(directory, "curl.log"), + CCH_FAKE_DOCKER_LOG: path.join(directory, "docker.log"), + ...overrides, + }, + } + ); +} + +afterEach(async () => { + const exits = [...children].map( + (child) => + new Promise((resolve) => { + if (child.exitCode !== null || child.signalCode !== null) { + resolve(); + return; + } + const forceTimer = setTimeout(() => { + if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); + }, 1000); + child.once("exit", () => { + clearTimeout(forceTimer); + resolve(); + }); + child.kill("SIGTERM"); + }) + ); + children.clear(); + await Promise.all(exits); +}); + +describe("issue #1408 load fixture", () => { + it("keeps repository scripts syntactically valid and independent from temporary paths", () => { + for (const filename of nodeScripts) { + const file = path.join(fixtureDir, filename); + execFileSync(process.execPath, ["--check", file]); + expect(readFileSync(file, "utf8")).not.toContain("/private/tmp"); + } + + for (const filename of shellScripts) { + const file = path.join(fixtureDir, filename); + if (process.platform !== "win32") execFileSync("sh", ["-n", file]); + expect(readFileSync(file, "utf8")).not.toContain("/private/tmp"); + } + + const startMock = readFileSync(path.join(fixtureDir, "start-mock-container.sh"), "utf8"); + expect(startMock).toContain('docker container inspect "$container"'); + }); + + posixIt("checks container existence without treating a same-named image as a collision", () => { + const directory = mkdtempSync(path.join(tmpdir(), "cch1408-container-fixture-")); + try { + installFakeContainerCommands(directory); + const result = runStartMockWithFakeCommands(directory, { CCH_FAKE_CURL_STATUS: "0" }); + + expect(result.status).toBe(0); + expect(result.stdout).toContain("ready container=fixture-container"); + expect(existsSync(path.join(directory, "container-state"))).toBe(true); + expect(readFileSync(path.join(directory, "docker.log"), "utf8")).toMatch( + /^container inspect fixture-container$/m + ); + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }); + + posixIt.each(["signal-int", "signal-term", "fail"])( + "removes the new container when readiness exits via %s", + (sleepMode) => { + const directory = mkdtempSync(path.join(tmpdir(), "cch1408-container-fixture-")); + try { + installFakeContainerCommands(directory); + const result = runStartMockWithFakeCommands(directory, { + CCH_FAKE_CURL_STATUS: "1", + CCH_FAKE_SLEEP_MODE: sleepMode, + }); + + expect(result.status).not.toBe(0); + expect(existsSync(path.join(directory, "container-state"))).toBe(false); + expect(readFileSync(path.join(directory, "docker.log"), "utf8")).toMatch( + /^rm -f fixture-container$/m + ); + } finally { + rmSync(directory, { recursive: true, force: true }); + } + } + ); + + posixIt("bounds stalled health probes and cleans up after the retry budget", () => { + const directory = mkdtempSync(path.join(tmpdir(), "cch1408-container-fixture-")); + try { + installFakeContainerCommands(directory); + const result = runStartMockWithFakeCommands(directory, { + CCH_FAKE_CURL_STATUS: "28", + }); + + expect(result.status).not.toBe(0); + expect(existsSync(path.join(directory, "container-state"))).toBe(false); + const probes = readFileSync(path.join(directory, "curl.log"), "utf8").trim().split("\n"); + expect(probes).toHaveLength(60); + expect(new Set(probes)).toEqual( + new Set(["--connect-timeout 2 --max-time 5 -fsS http://127.0.0.1:31409/health"]) + ); + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }); + + it("emits a valid hanging Responses SSE stream and records the scenario count", async () => { + const { baseUrl } = await startMock(); + await postAndAbort( + new URL("/v1/responses", baseUrl), + JSON.stringify({ input: "CCH_SCENARIO_contract-fixture", stream: true }) + ); + + const stats = await getJson(new URL("/stats", baseUrl)); + expect(stats).toMatchObject({ + counts: { "contract-fixture": 1 }, + totalMiB: 0.0625, + }); + }); + + it("ignores inherited mock configuration when starting the fixture", async () => { + const previous = process.env.CCH_MOCK_MAX_REQUEST_BYTES; + process.env.CCH_MOCK_MAX_REQUEST_BYTES = "1.5"; + try { + const { baseUrl } = await startMock(); + await expect(getJson(new URL("/stats", baseUrl))).resolves.toMatchObject({ counts: {} }); + } finally { + if (previous === undefined) delete process.env.CCH_MOCK_MAX_REQUEST_BYTES; + else process.env.CCH_MOCK_MAX_REQUEST_BYTES = previous; + } + }); + + it("resets counters, rejects unknown routes, and bounds request bodies", async () => { + const { baseUrl } = await startMock({ CCH_MOCK_MAX_REQUEST_BYTES: "64" }); + + const missing = await requestJson(new URL("/unknown", baseUrl), "GET"); + expect(missing).toEqual({ statusCode: 404, body: { error: "not found" } }); + + const oversized = await requestJson( + new URL("/v1/responses", baseUrl), + "POST", + JSON.stringify({ input: "x".repeat(128), stream: true }) + ); + expect(oversized).toEqual({ + statusCode: 413, + body: { error: "request body too large" }, + }); + + await postAndAbort( + new URL("/v1/responses", baseUrl), + JSON.stringify({ input: "CCH_SCENARIO_reset-me", stream: true }) + ); + await expect(getJson(new URL("/stats", baseUrl))).resolves.toMatchObject({ + counts: { "reset-me": 1 }, + }); + + const reset = await requestJson(new URL("/reset", baseUrl), "POST"); + expect(reset).toEqual({ statusCode: 200, body: { reset: true } }); + await expect(getJson(new URL("/stats", baseUrl))).resolves.toMatchObject({ counts: {} }); + }); + + it.each([ + ["payload below one frame", { CCH_MOCK_MIB: "0.01" }, "CCH_MOCK_MIB"], + ["payload above the fixture cap", { CCH_MOCK_MIB: "65" }, "CCH_MOCK_MIB"], + ["fractional request limit", { CCH_MOCK_MAX_REQUEST_BYTES: "1.5" }, "CCH_MOCK_MAX"], + ])("rejects invalid mock configuration: %s", (_name, overrides, expected) => { + const result = spawnSync(process.execPath, [path.join(fixtureDir, "mock-upstream.cjs")], { + encoding: "utf8", + env: createMockEnvironment(overrides), + }); + + expect(result.status).not.toBe(0); + expect(result.stderr).toContain(expected); + }); + + it("prints a structured memory sample when preloaded", () => { + const output = execFileSync(process.execPath, [path.join(fixtureDir, "memory-probe.cjs")], { + encoding: "utf8", + }); + const sample = JSON.parse(output.trim()) as Record; + expect(sample).toMatchObject({ cchMemoryProbe: true }); + expect(sample.rssMiB).toEqual(expect.any(Number)); + expect(sample.heapUsedMiB).toEqual(expect.any(Number)); + expect(sample.externalMiB).toEqual(expect.any(Number)); + expect(sample.arrayBuffersMiB).toEqual(expect.any(Number)); + expect(sample.resources).toEqual(expect.any(Object)); + }); + + it("requires the API key through an explicit environment boundary", () => { + const result = spawnSync( + process.execPath, + [ + path.join(fixtureDir, "drive-disconnect-waves.cjs"), + "http://127.0.0.1:1", + "http://127.0.0.1:2/stats", + "missing-key", + "1", + "1", + "0", + ], + { + encoding: "utf8", + env: { ...process.env, CCH_API_KEY: "", CCH_API_KEY_FILE: "" }, + } + ); + + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("set CCH_API_KEY or CCH_API_KEY_FILE"); + }); + + it("fails when the mock stats response is interrupted after headers", async () => { + const server = http.createServer((_request, response) => { + response.writeHead(200, { "content-type": "application/json" }); + response.write('{"counts":'); + setTimeout(() => response.destroy(), 10); + }); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }); + + try { + const address = server.address(); + if (!address || typeof address === "string") throw new Error("fixture server has no port"); + const child = spawn( + process.execPath, + [ + path.join(fixtureDir, "drive-disconnect-waves.cjs"), + "http://127.0.0.1:1", + `http://127.0.0.1:${address.port}/stats`, + "interrupted-stats", + "1", + "1", + "0", + ], + { + env: { ...process.env, CCH_API_KEY: "fixture-key", CCH_API_KEY_FILE: "" }, + stdio: ["ignore", "pipe", "pipe"], + } + ); + children.add(child); + + const result = await waitForExit(child); + expect(result.code).not.toBe(0); + expect(result.stderr).toContain("response aborted"); + } finally { + await new Promise((resolve) => server.close(() => resolve())); + } + }); + + it.each([ + ["unsupported URL protocol", ["ftp://127.0.0.1", "http://127.0.0.1/stats", "valid"], "APP_URL"], + [ + "invalid scenario characters", + ["http://127.0.0.1", "http://127.0.0.1/stats", "bad:value"], + "SCENARIO_PREFIX", + ], + ["zero waves", ["http://127.0.0.1", "http://127.0.0.1/stats", "valid", "0"], "WAVES"], + ])("rejects invalid driver input: %s", (_name, args, expected) => { + const result = spawnSync( + process.execPath, + [path.join(fixtureDir, "drive-disconnect-waves.cjs"), ...args], + { + encoding: "utf8", + env: { ...process.env, CCH_API_KEY: "fixture-key", CCH_API_KEY_FILE: "" }, + } + ); + + expect(result.status).not.toBe(0); + expect(result.stderr).toContain(expected); + }); +}); diff --git a/tests/unit/proxy/replay-spool.test.ts b/tests/unit/proxy/replay-spool.test.ts index e366083d2..f07e67408 100644 --- a/tests/unit/proxy/replay-spool.test.ts +++ b/tests/unit/proxy/replay-spool.test.ts @@ -29,7 +29,7 @@ const envControl = vi.hoisted(() => ({ const storeControl = vi.hoisted(() => { const order: string[] = []; - let ownedChunkCount = 0; + const ownedChunksByReplayId = new Map(); const store = { appendChunks: vi.fn(async (_replayId: string, values: string[]) => { order.push(`append:${values.join("|")}`); @@ -41,16 +41,22 @@ const storeControl = vi.hoisted(() => { }), writeOwned: vi.fn( async ( - _replayId: string, + replayId: string, _ownerToken: string, _meta: { status: string }, values: string[] = [] ) => { order.push(`write:${values.join("|")}`); - ownedChunkCount += values.length; - return ownedChunkCount; + const ownedChunks = ownedChunksByReplayId.get(replayId) ?? []; + ownedChunks.push(...values); + ownedChunksByReplayId.set(replayId, ownedChunks); + return ownedChunks.length; } ), + readChunks: vi.fn(async (replayId: string, fromIndex: number) => { + order.push("read"); + return (ownedChunksByReplayId.get(replayId) ?? []).slice(fromIndex); + }), completeOwned: vi.fn( async (_replayId: string, _ownerToken: string, meta: { status: string }) => { order.push(`meta:${meta.status}`); @@ -89,8 +95,8 @@ const storeControl = vi.hoisted(() => { return { order, store, - resetOwnedChunkCount: () => { - ownedChunkCount = 0; + resetOwnedChunks: () => { + ownedChunksByReplayId.clear(); }, }; }); @@ -156,13 +162,6 @@ async function drainWriteChain(spool: ReplaySpool): Promise { await (spool as unknown as { writeChain: Promise }).writeChain; } -function retainedAsciiPartBytes(spool: ReplaySpool): number { - return (spool as unknown as { parts: string[] }).parts.reduce( - (total, part) => total + part.length, - 0 - ); -} - function makeOwnerSession(): ProxySession { return { replayState: { identity, ownerToken: "owner-token", role: "owner" }, @@ -181,7 +180,7 @@ beforeEach(() => { envControl.maxPayloadBytes = 8 * 1024 * 1024; envControl.maxConcurrentSpools = 64; storeControl.order.length = 0; - storeControl.resetOwnedChunkCount(); + storeControl.resetOwnedChunks(); storeControl.store.abortOwned.mockImplementation( async (_replayId: string, _ownerToken: string, meta: { status: string }) => { storeControl.order.push(`meta:${meta.status}`); @@ -191,6 +190,11 @@ beforeEach(() => { } ); storeControl.store.completeOwned.mockClear(); + storeControl.store.persistCompleted.mockReset(); + storeControl.store.persistCompleted.mockImplementation(async () => { + storeControl.order.push("persist"); + return "persisted" as const; + }); storeControl.store.releaseOwner.mockImplementation(async () => { storeControl.order.push("release"); }); @@ -256,6 +260,28 @@ describe("ReplaySpool：write-behind 批量冲刷", () => { await spool.abort("test_cleanup"); }); + it("跨 chunk UTF-8 序列按全部输入字节触发冲刷阈值", async () => { + const spool = makeSpool(); + const continuation = new Uint8Array(1 + (64 * 1024 - 3)); + continuation[0] = 0xad; + continuation.fill(0x78, 1); + + spool.observe(new Uint8Array([0xe4, 0xb8])); + expect((spool as unknown as { pendingBytes: number }).pendingBytes).toBe(2); + + spool.observe(continuation); + expect((spool as unknown as { queuedWriteBytes: number }).queuedWriteBytes).toBe(64 * 1024); + + await drainWriteChain(spool); + expect(storeControl.store.writeOwned).toHaveBeenCalledWith( + identity.replayId, + "owner-token", + expect.objectContaining({ byteSize: 64 * 1024 }), + [`中${"x".repeat(64 * 1024 - 3)}`] + ); + await spool.abort("test_cleanup"); + }); + it("没有响应 chunk 时仍按 15 秒间隔续租 owner", async () => { const spool = makeSpool(); @@ -324,6 +350,49 @@ describe("ReplaySpool：write-behind 批量冲刷", () => { expect(spool.isTerminal).toBe(true); expect(getActiveReplaySpoolCount()).toBe(0); }); + + it("Redis write backlog 超过 1 MiB 时放弃 spool 并清空排队 batch", async () => { + let resolveFirstWrite!: (value: number) => void; + const onInactive = vi.fn(); + storeControl.store.writeOwned.mockImplementationOnce( + () => + new Promise((resolve) => { + resolveFirstWrite = resolve; + }) + ); + const spool = new ReplaySpool( + identity, + "owner-token", + 200, + { "content-type": "text/event-stream" }, + "stream", + { onInactive } + ); + + for (let index = 0; index < 16; index += 1) { + spool.observe(encoder.encode("x".repeat(64 * 1024))); + await vi.advanceTimersByTimeAsync(0); + } + + expect(spool.isTerminal).toBe(false); + expect((spool as unknown as { queuedWriteBytes: number }).queuedWriteBytes).toBe(1024 * 1024); + + spool.observe(encoder.encode("x".repeat(64 * 1024))); + await vi.advanceTimersByTimeAsync(0); + + expect(spool.isTerminal).toBe(true); + expect(onInactive).toHaveBeenCalledTimes(1); + expect((spool as unknown as { queuedBatches: Set }).queuedBatches.size).toBe(0); + expect((spool as unknown as { queuedWriteBytes: number }).queuedWriteBytes).toBe(0); + + resolveFirstWrite(1); + await drainWriteChain(spool); + expect(storeControl.store.abortOwned).toHaveBeenCalledWith( + identity.replayId, + "owner-token", + expect.objectContaining({ status: "aborted", abortReason: "write_backlog_too_large" }) + ); + }); }); describe("ReplaySpool：续租丢失 halt", () => { @@ -403,6 +472,7 @@ describe("ReplaySpool：completeAfterBilling 终态屏障", () => { expect(storeControl.order).toEqual([ "write:data: hello \n\n|data: world\n\n", + "read", "persist", "meta:completed", "release", @@ -496,6 +566,41 @@ describe("ReplaySpool：completeAfterBilling 终态屏障", () => { expect(getActiveReplaySpoolCount()).toBe(0); }); + it("阻塞写入加完成尾批超过 1 MiB 时放弃 spool 并等待 fenced cleanup", async () => { + let resolveFirstWrite!: (value: number) => void; + storeControl.store.writeOwned.mockImplementationOnce( + () => + new Promise((resolve) => { + resolveFirstWrite = resolve; + }) + ); + const spool = makeSpool(); + for (let index = 0; index < 16; index += 1) { + spool.observe(encoder.encode("x".repeat(64 * 1024))); + await vi.advanceTimersByTimeAsync(0); + } + expect((spool as unknown as { queuedWriteBytes: number }).queuedWriteBytes).toBe(1024 * 1024); + + spool.observe(encoder.encode("tail")); + const completion = spool.completeAfterBilling(5); + await vi.advanceTimersByTimeAsync(0); + + expect(spool.isTerminal).toBe(true); + expect((spool as unknown as { pending: string[] }).pending).toEqual([]); + expect((spool as unknown as { queuedBatches: Set }).queuedBatches.size).toBe(0); + expect((spool as unknown as { queuedWriteBytes: number }).queuedWriteBytes).toBe(0); + expect(storeControl.store.persistCompleted).not.toHaveBeenCalled(); + + resolveFirstWrite(1); + await completion; + expect(storeControl.store.abortOwned).toHaveBeenCalledWith( + identity.replayId, + "owner-token", + expect.objectContaining({ status: "aborted", abortReason: "write_backlog_too_large" }) + ); + expect(getActiveReplaySpoolCount()).toBe(0); + }); + it("persist 成功但 completed 翻转失败：日志标记 pgPersisted=true，热层封死为 aborted", async () => { storeControl.store.completeOwned.mockResolvedValueOnce(false); const spool = makeSpool(); @@ -540,17 +645,10 @@ describe("ReplaySpool：completeAfterBilling 终态屏障", () => { expect(storeControl.store.completeOwned).toHaveBeenCalledTimes(1); }); - it("Redis 阻塞期间不提前复制 payload，PG 阻塞期间释放 parts", async () => { + it("活跃 spool 不保留整流 parts，完成时从 Redis chunks 组装 durable payload", async () => { const payloadBytes = 4 * 1024 * 1024; const chunk = "x".repeat(64 * 1024); - let resolveRedis!: (value: number) => void; let resolvePersist!: (value: "persisted") => void; - storeControl.store.writeOwned.mockImplementationOnce( - () => - new Promise((resolve) => { - resolveRedis = resolve; - }) - ); storeControl.store.persistCompleted.mockImplementationOnce( () => new Promise<"persisted">((resolve) => { @@ -560,41 +658,79 @@ describe("ReplaySpool：completeAfterBilling 终态屏障", () => { const spool = makeSpool(); for (let index = 0; index < 64; index += 1) { spool.observe(encoder.encode(chunk)); + await vi.advanceTimersByTimeAsync(0); + await drainWriteChain(spool); } - expect(retainedAsciiPartBytes(spool)).toBe(payloadBytes); + expect("parts" in (spool as object)).toBe(false); const completion = spool.completeAfterBilling(10); await vi.advanceTimersByTimeAsync(0); - expect(storeControl.store.writeOwned).toHaveBeenCalledTimes(1); - expect(storeControl.store.persistCompleted).not.toHaveBeenCalled(); - expect(retainedAsciiPartBytes(spool)).toBe(payloadBytes); - - resolveRedis(1); - await vi.advanceTimersByTimeAsync(0); - expect(storeControl.store.persistCompleted).toHaveBeenCalledTimes(1); + expect(storeControl.store.readChunks).toHaveBeenCalledWith(identity.replayId, 0); expect(storeControl.store.persistCompleted).toHaveBeenCalledWith( expect.objectContaining({ payload: "x".repeat(payloadBytes), byteSize: payloadBytes, }) ); - expect(retainedAsciiPartBytes(spool)).toBe(0); - resolvePersist("persisted"); await completion; }); - it("payload 组装失败时封死热层并释放 heartbeat 与并发配额", async () => { + it("并发完成时串行重建和持久化完整 payload，限制瞬时堆峰值", async () => { + const secondIdentity: ReplayIdentity = { + ...identity, + replayId: "1123456789abcdef0123456789abcdef", + verifier: "eedcba9876543210fedcba9876543210", + }; + let resolveFirstPersist!: (value: "persisted") => void; + storeControl.store.persistCompleted.mockImplementation((row: { replayId: string }) => { + if (row.replayId === identity.replayId) { + return new Promise<"persisted">((resolve) => { + resolveFirstPersist = resolve; + }); + } + return Promise.resolve("persisted" as const); + }); + const first = makeSpool(); + const second = new ReplaySpool( + secondIdentity, + "second-owner-token", + 200, + { "content-type": "text/event-stream" }, + "stream" + ); + first.observe(encoder.encode("data: first\n\n")); + second.observe(encoder.encode("data: second\n\n")); + + const firstCompletion = first.completeAfterBilling(10); + const secondCompletion = second.completeAfterBilling(11); + await vi.advanceTimersByTimeAsync(0); + + expect(storeControl.store.persistCompleted).toHaveBeenCalledTimes(1); + expect(storeControl.store.persistCompleted).toHaveBeenCalledWith( + expect.objectContaining({ replayId: identity.replayId, payload: "data: first\n\n" }) + ); + + resolveFirstPersist("persisted"); + await firstCompletion; + await vi.advanceTimersByTimeAsync(0); + await secondCompletion; + + expect(storeControl.store.persistCompleted).toHaveBeenCalledTimes(2); + expect(storeControl.store.persistCompleted).toHaveBeenLastCalledWith( + expect.objectContaining({ + replayId: secondIdentity.replayId, + payload: "data: second\n\n", + }) + ); + }); + + it("Redis chunks 缺失时封死热层并释放 heartbeat 与并发配额", async () => { const spool = makeSpool(); spool.observe(encoder.encode("data: partial\n\n")); - const parts = (spool as unknown as { parts: unknown[] }).parts; - parts[0] = { - toString: () => { - throw new Error("payload assembly failed"); - }, - }; + storeControl.store.readChunks.mockResolvedValueOnce(null); await spool.completeAfterBilling(10); @@ -610,6 +746,36 @@ describe("ReplaySpool：completeAfterBilling 终态屏障", () => { expect(storeControl.store.renewOwnerLease).not.toHaveBeenCalled(); }); + it("inactive 回调异常时仍完成 spool 清理并释放并发配额", async () => { + const onInactive = vi.fn(() => { + throw new Error("callback failed"); + }); + const spool = new ReplaySpool( + identity, + "owner-token", + 200, + { "content-type": "text/event-stream" }, + "stream", + { onInactive } + ); + envControl.maxPayloadBytes = 4; + + spool.observe(encoder.encode("12345678")); + await drainWriteChain(spool); + + expect(onInactive).toHaveBeenCalledOnce(); + expect(storeControl.store.abortOwned).toHaveBeenCalledWith( + identity.replayId, + "owner-token", + expect.objectContaining({ abortReason: "payload_too_large" }) + ); + expect(logger.debug).toHaveBeenCalledWith( + "[ReplaySpool] inactive callback failed", + expect.objectContaining({ error: "callback failed" }) + ); + expect(getActiveReplaySpoolCount()).toBe(0); + }); + it("跨 chunk 截断的 UTF-8 序列在 complete 时冲刷解码尾部", async () => { const spool = makeSpool(); // "中" (0xE4 0xB8 0xAD) 只送前两字节：observe 阶段解码挂起，complete 时 flush 出替换字符 @@ -656,13 +822,14 @@ describe("ReplaySpool：abort 终态", () => { expect(getActiveReplaySpoolCount()).toBe(0); }); - it("abort 立即释放已累积的 payload", async () => { + it("abort 立即清空 pending 与 queued batch", async () => { const spool = makeSpool(); spool.observe(encoder.encode("data: partial\n\n")); await spool.abort("upstream_error"); - expect((spool as unknown as { parts: string[] }).parts).toEqual([]); + expect((spool as unknown as { pending: string[] }).pending).toEqual([]); + expect((spool as unknown as { queuedBatches: Set }).queuedBatches.size).toBe(0); }); it("Redis flush 阻塞时 abort 立即释放 batch，并在 fenced cleanup 后释放并发配额", async () => { @@ -798,10 +965,22 @@ describe("ReplaySpool：isTerminal", () => { expect(aborted.isTerminal).toBe(true); envControl.maxPayloadBytes = 4; - const oversized = makeSpool(); + const onInactive = vi.fn(); + const oversized = new ReplaySpool( + identity, + "owner-token", + 200, + { "content-type": "text/event-stream" }, + "stream", + { onInactive } + ); oversized.observe(encoder.encode("12345678")); expect(oversized.isTerminal).toBe(true); + expect(onInactive).toHaveBeenCalledTimes(1); await drainWriteChain(oversized); + + await oversized.abort("late_abort"); + expect(onInactive).toHaveBeenCalledTimes(1); }); }); diff --git a/tests/unit/proxy/response-handler-stream-terminal.test.ts b/tests/unit/proxy/response-handler-stream-terminal.test.ts index 8486af410..3718a28c3 100644 --- a/tests/unit/proxy/response-handler-stream-terminal.test.ts +++ b/tests/unit/proxy/response-handler-stream-terminal.test.ts @@ -19,6 +19,7 @@ const mocks = vi.hoisted(() => ({ replayObserve: vi.fn(), replayComplete: vi.fn(async () => {}), replayAbort: vi.fn(async () => {}), + replayInactive: null as (() => void) | null, })); vi.mock("@/app/v1/_lib/proxy/response-fixer", () => ({ @@ -60,15 +61,21 @@ vi.mock("@/lib/proxy-status-tracker", () => ({ })); vi.mock("@/app/v1/_lib/proxy/replay/replay-spool", () => ({ abortReplayOwnership: vi.fn(async () => undefined), - createReplaySpoolIfOwner: (session: ProxySession) => - session.replayState?.role === "owner" - ? { - abort: mocks.replayAbort, - completeAfterBilling: mocks.replayComplete, - isTerminal: false, - observe: mocks.replayObserve, - } - : null, + createReplaySpoolIfOwner: ( + session: ProxySession, + _response: Response, + _delivery: string, + options: { onInactive?: () => void } = {} + ) => { + if (session.replayState?.role !== "owner") return null; + mocks.replayInactive = options.onInactive ?? null; + return { + abort: mocks.replayAbort, + completeAfterBilling: mocks.replayComplete, + isTerminal: false, + observe: mocks.replayObserve, + }; + }, releaseReplayOwnership: vi.fn(), })); vi.mock("@/repository/message", () => ({ @@ -222,9 +229,27 @@ function sseResponse(body: BodyInit, status = 200): Response { return new Response(body, { status, headers: { "content-type": "text/event-stream" } }); } +function setReplayOwner(session: ProxySession, suffix: string): void { + session.replayState = { + role: "owner", + ownerToken: `owner-token-${suffix}`, + identity: { + replayId: `replay-${suffix}`, + verifier: "verifier", + scopeTag: "scope-tag", + keyId: KEY.id, + userId: USER.id, + format: "claude", + model: "claude-test", + endpoint: "/v1/messages", + }, + }; +} + describe("ProxyResponseHandler.dispatch stream terminal behavior", () => { beforeEach(() => { mocks.tasks.length = 0; + mocks.replayInactive = null; vi.clearAllMocks(); mocks.durable.mockImplementation(async (_id, _details, options) => { await options?.onCommitted?.(); @@ -277,6 +302,131 @@ describe("ProxyResponseHandler.dispatch stream terminal behavior", () => { expect(releaseAgent).toHaveBeenCalledOnce(); }); + it("caps a detached Replay drain at 60 seconds after the spool becomes inactive", async () => { + vi.useFakeTimers(); + const previousReplayDetachedMs = process.env.REPLAY_MAX_DETACHED_MS; + process.env.REPLAY_MAX_DETACHED_MS = "300000"; + try { + const cancelSource = vi.fn(); + const responseController = new AbortController(); + const source = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('data: {"partial":true}\n\n')); + }, + cancel: cancelSource, + }); + const { session } = await createSession({ responseController }); + setReplayOwner(session, "detached"); + + const returned = await ProxyResponseHandler.dispatch(session, sseResponse(source)); + const reader = returned.body?.getReader(); + await reader?.read(); + await reader?.cancel(new Error("client disconnected")); + + expect(mocks.replayInactive).toEqual(expect.any(Function)); + await vi.advanceTimersByTimeAsync(59_999); + expect(responseController.signal.aborted).toBe(false); + + mocks.replayInactive?.(); + await vi.advanceTimersByTimeAsync(1); + + expect(responseController.signal.aborted).toBe(true); + expect(responseController.signal.reason).toEqual( + expect.objectContaining({ message: "client_abort_drain_timeout" }) + ); + expect(cancelSource).toHaveBeenCalledOnce(); + await settleTasks(); + } finally { + if (previousReplayDetachedMs === undefined) { + delete process.env.REPLAY_MAX_DETACHED_MS; + } else { + process.env.REPLAY_MAX_DETACHED_MS = previousReplayDetachedMs; + } + vi.useRealTimers(); + } + }); + + it("keeps the configured 300-second drain while the Replay spool remains active", async () => { + vi.useFakeTimers(); + const previousReplayDetachedMs = process.env.REPLAY_MAX_DETACHED_MS; + process.env.REPLAY_MAX_DETACHED_MS = "300000"; + try { + const cancelSource = vi.fn(); + const responseController = new AbortController(); + const source = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('data: {"partial":true}\n\n')); + }, + cancel: cancelSource, + }); + const { session } = await createSession({ responseController }); + setReplayOwner(session, "active"); + + const returned = await ProxyResponseHandler.dispatch(session, sseResponse(source)); + const reader = returned.body?.getReader(); + await reader?.read(); + await reader?.cancel(new Error("client disconnected")); + + await vi.advanceTimersByTimeAsync(60_000); + expect(responseController.signal.aborted).toBe(false); + await vi.advanceTimersByTimeAsync(239_999); + expect(responseController.signal.aborted).toBe(false); + + await vi.advanceTimersByTimeAsync(1); + + expect(responseController.signal.aborted).toBe(true); + expect(cancelSource).toHaveBeenCalledOnce(); + await settleTasks(); + } finally { + if (previousReplayDetachedMs === undefined) { + delete process.env.REPLAY_MAX_DETACHED_MS; + } else { + process.env.REPLAY_MAX_DETACHED_MS = previousReplayDetachedMs; + } + vi.useRealTimers(); + } + }); + + it("uses the 60-second drain when Replay becomes inactive before client detach", async () => { + vi.useFakeTimers(); + const previousReplayDetachedMs = process.env.REPLAY_MAX_DETACHED_MS; + process.env.REPLAY_MAX_DETACHED_MS = "300000"; + try { + const cancelSource = vi.fn(); + const responseController = new AbortController(); + const source = new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('data: {"partial":true}\n\n')); + }, + cancel: cancelSource, + }); + const { session } = await createSession({ responseController }); + setReplayOwner(session, "inactive-before-detach"); + + const returned = await ProxyResponseHandler.dispatch(session, sseResponse(source)); + const reader = returned.body?.getReader(); + await reader?.read(); + expect(mocks.replayInactive).toEqual(expect.any(Function)); + mocks.replayInactive?.(); + await reader?.cancel(new Error("client disconnected")); + + await vi.advanceTimersByTimeAsync(59_999); + expect(responseController.signal.aborted).toBe(false); + await vi.advanceTimersByTimeAsync(1); + + expect(responseController.signal.aborted).toBe(true); + expect(cancelSource).toHaveBeenCalledOnce(); + await settleTasks(); + } finally { + if (previousReplayDetachedMs === undefined) { + delete process.env.REPLAY_MAX_DETACHED_MS; + } else { + process.env.REPLAY_MAX_DETACHED_MS = previousReplayDetachedMs; + } + vi.useRealTimers(); + } + }); + it("persists a response-controller timeout as 502 and cancels the source", async () => { const cancelSource = vi.fn(); const responseController = new AbortController(); From dc3c2a8925b18e5f83a7ba2cd30a8e2a7b125d9e Mon Sep 17 00:00:00 2001 From: LAMCLOD <2070346656@qq.com> Date: Wed, 12 Aug 2026 15:49:07 +0800 Subject: [PATCH 09/12] =?UTF-8?q?feat(availability):=20=E5=8F=AF=E7=94=A8?= =?UTF-8?q?=E6=80=A7=E7=9B=91=E6=8E=A7=20outbox=20+=201=20=E5=88=86?= =?UTF-8?q?=E9=92=9F=E6=8A=95=E5=BD=B1=E6=A1=B6=20(#1416)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(availability): outbox + 1m projection buckets for admin monitoring Replace on-the-fly message_request scans with trigger/outbox-fed avail_bucket_1m/avail_current so availability APIs stay fast under load. * fix(availability): address review on projection freshness and worker safety * fix(availability): stable provider lock order in projection worker Sort provider/bucket ids before upsert and recompute so concurrent worker instances take avail_current locks in the same order. --- drizzle/0120_availability_projection.sql | 135 + drizzle/meta/0120_snapshot.json | 5745 +++++++++++++++++ drizzle/meta/_journal.json | 9 +- src/drizzle/schema.ts | 10 + src/instrumentation.ts | 22 + src/lib/availability/availability-service.ts | 398 +- src/lib/availability/index.ts | 8 +- src/lib/availability/projection-tables.ts | 85 + src/lib/availability/projection-worker.ts | 540 ++ src/lib/availability/types.ts | 10 +- src/lib/lifecycle/shutdown.ts | 21 +- tests/unit/lib/availability-service.test.ts | 325 +- .../availability/projection-worker.test.ts | 210 + tests/unit/lib/shutdown.test.ts | 20 + tests/unit/server-shutdown.test.ts | 3 + 15 files changed, 7135 insertions(+), 406 deletions(-) create mode 100644 drizzle/0120_availability_projection.sql create mode 100644 drizzle/meta/0120_snapshot.json create mode 100644 src/lib/availability/projection-tables.ts create mode 100644 src/lib/availability/projection-worker.ts create mode 100644 tests/unit/lib/availability/projection-worker.test.ts diff --git a/drizzle/0120_availability_projection.sql b/drizzle/0120_availability_projection.sql new file mode 100644 index 000000000..b0207db51 --- /dev/null +++ b/drizzle/0120_availability_projection.sql @@ -0,0 +1,135 @@ +-- Availability projection: outbox + 1-minute buckets (read path no longer scans message_request) +CREATE EXTENSION IF NOT EXISTS pgcrypto;--> statement-breakpoint + +CREATE TABLE IF NOT EXISTS "outbox_events" ( + "id" bigserial PRIMARY KEY, + "event_id" uuid DEFAULT gen_random_uuid() NOT NULL, + "event_type" text NOT NULL, + "aggregate_type" text NOT NULL, + "aggregate_id" bigint NOT NULL, + "occurred_at" timestamp with time zone NOT NULL, + "payload" jsonb NOT NULL, + "created_at" timestamp with time zone DEFAULT now() NOT NULL, + "published_at" timestamp with time zone, + "attempts" integer DEFAULT 0 NOT NULL, + "last_error" text, + CONSTRAINT "outbox_events_event_id_key" UNIQUE("event_id") +);--> statement-breakpoint + +CREATE INDEX IF NOT EXISTS "idx_outbox_events_unpublished" + ON "outbox_events" USING btree ("id" ASC) + WHERE "published_at" IS NULL;--> statement-breakpoint + +CREATE TABLE IF NOT EXISTS "outbox_processed" ( + "event_id" uuid PRIMARY KEY, + "processed_at" timestamp with time zone DEFAULT now() NOT NULL +);--> statement-breakpoint + +CREATE TABLE IF NOT EXISTS "proj_applied_requests" ( + "request_id" bigint PRIMARY KEY, + "event_id" uuid NOT NULL, + "applied_at" timestamp with time zone DEFAULT now() NOT NULL +);--> statement-breakpoint + +CREATE TABLE IF NOT EXISTS "avail_bucket_1m" ( + "provider_id" integer NOT NULL, + "bucket_start" timestamp with time zone NOT NULL, + "success_cnt" integer DEFAULT 0 NOT NULL, + "failure_cnt" integer DEFAULT 0 NOT NULL, + "excluded_cnt" integer DEFAULT 0 NOT NULL, + "latency_cnt" integer DEFAULT 0 NOT NULL, + "latency_sum_ms" bigint DEFAULT 0 NOT NULL, + "last_request_at" timestamp with time zone, + CONSTRAINT "avail_bucket_1m_pkey" PRIMARY KEY("provider_id","bucket_start") +);--> statement-breakpoint + +CREATE INDEX IF NOT EXISTS "idx_avail_bucket_1m_time" + ON "avail_bucket_1m" USING btree ("bucket_start" DESC);--> statement-breakpoint + +CREATE TABLE IF NOT EXISTS "avail_current" ( + "provider_id" integer PRIMARY KEY, + "state" text DEFAULT 'unknown' NOT NULL, + "availability" double precision DEFAULT 0 NOT NULL, + "request_count" integer DEFAULT 0 NOT NULL, + "last_request_at" timestamp with time zone, + "updated_at" timestamp with time zone DEFAULT now() NOT NULL +);--> statement-breakpoint + +CREATE TABLE IF NOT EXISTS "projection_meta" ( + "key" text PRIMARY KEY, + "value" jsonb DEFAULT '{}'::jsonb NOT NULL, + "updated_at" timestamp with time zone DEFAULT now() NOT NULL +);--> statement-breakpoint + +INSERT INTO "projection_meta" ("key", "value") +VALUES ('bootstrap', jsonb_build_object('version', 1, 'note', 'availability outbox projections')) +ON CONFLICT ("key") DO NOTHING;--> statement-breakpoint + +CREATE OR REPLACE FUNCTION trg_message_request_outbox() +RETURNS trigger +LANGUAGE plpgsql +AS $$ +DECLARE + v_outcome text; +BEGIN + IF NEW.status_code IS NULL THEN + RETURN NEW; + END IF; + + IF TG_OP = 'UPDATE' AND OLD.status_code IS NOT NULL THEN + RETURN NEW; + END IF; + + IF COALESCE(NEW.is_replay, false) THEN + RETURN NEW; + END IF; + + BEGIN + v_outcome := fn_compute_message_request_success_rate_outcome( + NEW.blocked_by, + NEW.status_code, + NEW.error_message, + NEW.provider_chain + ); + EXCEPTION WHEN OTHERS THEN + RAISE WARNING 'trg_message_request_outbox: outcome compute failed for request %: %', + NEW.id, SQLERRM; + v_outcome := NULL; + END; + + IF v_outcome IS NULL THEN + RETURN NEW; + END IF; + + INSERT INTO outbox_events ( + event_type, aggregate_type, aggregate_id, occurred_at, payload + ) VALUES ( + 'request_finalized', + 'message_request', + NEW.id, + COALESCE(NEW.created_at, now()), + jsonb_build_object( + 'request_id', NEW.id, + 'provider_id', NEW.provider_id, + 'model', NEW.model, + 'occurred_at', COALESCE(NEW.created_at, now()), + 'status_code', NEW.status_code, + 'duration_ms', NEW.duration_ms, + 'ttfb_ms', NEW.ttfb_ms, + 'blocked_by', NEW.blocked_by, + 'outcome', v_outcome, + 'group_tag', (SELECT group_tag FROM providers p WHERE p.id = NEW.provider_id), + 'is_replay', COALESCE(NEW.is_replay, false) + ) + ); + + RETURN NEW; +END; +$$;--> statement-breakpoint + +DROP TRIGGER IF EXISTS message_request_outbox_aiud ON message_request;--> statement-breakpoint +CREATE TRIGGER message_request_outbox_aiud + AFTER INSERT OR UPDATE OF status_code, duration_ms, error_message, provider_chain, blocked_by + ON message_request + FOR EACH ROW + EXECUTE FUNCTION trg_message_request_outbox(); diff --git a/drizzle/meta/0120_snapshot.json b/drizzle/meta/0120_snapshot.json new file mode 100644 index 000000000..728e3da75 --- /dev/null +++ b/drizzle/meta/0120_snapshot.json @@ -0,0 +1,5745 @@ +{ + "id": "5a73b354-be53-4ef5-bce8-e3eb4b3c3546", + "prevId": "12d909a0-d12b-4617-b7af-5a64a50201a7", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.audit_log": { + "name": "audit_log", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "action_category": { + "name": "action_category", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true + }, + "action_type": { + "name": "action_type", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true + }, + "target_type": { + "name": "target_type", + "type": "varchar(32)", + "primaryKey": false, + "notNull": false + }, + "target_id": { + "name": "target_id", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "target_name": { + "name": "target_name", + "type": "varchar(256)", + "primaryKey": false, + "notNull": false + }, + "before_value": { + "name": "before_value", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "after_value": { + "name": "after_value", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "operator_user_id": { + "name": "operator_user_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "operator_user_name": { + "name": "operator_user_name", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false + }, + "operator_key_id": { + "name": "operator_key_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "operator_key_name": { + "name": "operator_key_name", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false + }, + "operator_ip": { + "name": "operator_ip", + "type": "varchar(45)", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false + }, + "success": { + "name": "success", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_audit_log_category_created_at": { + "name": "idx_audit_log_category_created_at", + "columns": [ + { + "expression": "action_category", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_audit_log_operator_user_created_at": { + "name": "idx_audit_log_operator_user_created_at", + "columns": [ + { + "expression": "operator_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"audit_log\".\"operator_user_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_audit_log_operator_ip_created_at": { + "name": "idx_audit_log_operator_ip_created_at", + "columns": [ + { + "expression": "operator_ip", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"audit_log\".\"operator_ip\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_audit_log_target": { + "name": "idx_audit_log_target", + "columns": [ + { + "expression": "target_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"audit_log\".\"target_type\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_audit_log_created_at_id": { + "name": "idx_audit_log_created_at_id", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.cloud_pricing_catalog": { + "name": "cloud_pricing_catalog", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "version": { + "name": "version", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true + }, + "currency": { + "name": "currency", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "default": "'USD'" + }, + "refreshed_at": { + "name": "refreshed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "providers": { + "name": "providers", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "vendors": { + "name": "vendors", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "model_count": { + "name": "model_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "synced_at": { + "name": "synced_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.error_rules": { + "name": "error_rules", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "pattern": { + "name": "pattern", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "match_type": { + "name": "match_type", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'regex'" + }, + "category": { + "name": "category", + "type": "varchar(50)", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "override_response": { + "name": "override_response", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "override_status_code": { + "name": "override_status_code", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "is_enabled": { + "name": "is_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "is_default": { + "name": "is_default", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "priority": { + "name": "priority", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "idx_error_rules_enabled": { + "name": "idx_error_rules_enabled", + "columns": [ + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "priority", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "unique_pattern": { + "name": "unique_pattern", + "columns": [ + { + "expression": "pattern", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_category": { + "name": "idx_category", + "columns": [ + { + "expression": "category", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_match_type": { + "name": "idx_match_type", + "columns": [ + { + "expression": "match_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.keys": { + "name": "keys", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "is_enabled": { + "name": "is_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "can_login_web_ui": { + "name": "can_login_web_ui", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "limit_5h_usd": { + "name": "limit_5h_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "limit_5h_reset_mode": { + "name": "limit_5h_reset_mode", + "type": "daily_reset_mode", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'rolling'" + }, + "limit_daily_usd": { + "name": "limit_daily_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "daily_reset_mode": { + "name": "daily_reset_mode", + "type": "daily_reset_mode", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'fixed'" + }, + "daily_reset_time": { + "name": "daily_reset_time", + "type": "varchar(5)", + "primaryKey": false, + "notNull": true, + "default": "'00:00'" + }, + "limit_weekly_usd": { + "name": "limit_weekly_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "limit_monthly_usd": { + "name": "limit_monthly_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "limit_total_usd": { + "name": "limit_total_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "cost_reset_at": { + "name": "cost_reset_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "limit_concurrent_sessions": { + "name": "limit_concurrent_sessions", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "provider_group": { + "name": "provider_group", + "type": "varchar(200)", + "primaryKey": false, + "notNull": false, + "default": "'default'" + }, + "cache_ttl_preference": { + "name": "cache_ttl_preference", + "type": "varchar(10)", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "idx_keys_user_id": { + "name": "idx_keys_user_id", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_keys_key": { + "name": "idx_keys_key", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_keys_created_at": { + "name": "idx_keys_created_at", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_keys_deleted_at": { + "name": "idx_keys_deleted_at", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.message_request": { + "name": "message_request", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "model": { + "name": "model", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "cost_usd": { + "name": "cost_usd", + "type": "numeric(21, 15)", + "primaryKey": false, + "notNull": false, + "default": "'0'" + }, + "cost_multiplier": { + "name": "cost_multiplier", + "type": "numeric(10, 4)", + "primaryKey": false, + "notNull": false + }, + "group_cost_multiplier": { + "name": "group_cost_multiplier", + "type": "numeric(10, 4)", + "primaryKey": false, + "notNull": false + }, + "cost_breakdown": { + "name": "cost_breakdown", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "session_id": { + "name": "session_id", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "session_identity": { + "name": "session_identity", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "session_identity_kind": { + "name": "session_identity_kind", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "affinity_scope_tag": { + "name": "affinity_scope_tag", + "type": "varchar(16)", + "primaryKey": false, + "notNull": false + }, + "affinity_fingerprint": { + "name": "affinity_fingerprint", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "affinity_fingerprint_chain": { + "name": "affinity_fingerprint_chain", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "is_replay": { + "name": "is_replay", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "replay_source_request_id": { + "name": "replay_source_request_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "request_sequence": { + "name": "request_sequence", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 1 + }, + "provider_chain": { + "name": "provider_chain", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "routing_trace": { + "name": "routing_trace", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "status_code": { + "name": "status_code", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "api_type": { + "name": "api_type", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "endpoint": { + "name": "endpoint", + "type": "varchar(256)", + "primaryKey": false, + "notNull": false + }, + "original_model": { + "name": "original_model", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false + }, + "actual_response_model": { + "name": "actual_response_model", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false + }, + "input_tokens": { + "name": "input_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "output_tokens": { + "name": "output_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "ttfb_ms": { + "name": "ttfb_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "first_byte_ms": { + "name": "first_byte_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "cache_creation_input_tokens": { + "name": "cache_creation_input_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "cache_read_input_tokens": { + "name": "cache_read_input_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "cache_creation_5m_input_tokens": { + "name": "cache_creation_5m_input_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "cache_creation_1h_input_tokens": { + "name": "cache_creation_1h_input_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "cache_ttl_applied": { + "name": "cache_ttl_applied", + "type": "varchar(10)", + "primaryKey": false, + "notNull": false + }, + "context_1m_applied": { + "name": "context_1m_applied", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "swap_cache_ttl_applied": { + "name": "swap_cache_ttl_applied", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "special_settings": { + "name": "special_settings", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "hedge_losers": { + "name": "hedge_losers", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error_stack": { + "name": "error_stack", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error_cause": { + "name": "error_cause", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "blocked_by": { + "name": "blocked_by", + "type": "varchar(50)", + "primaryKey": false, + "notNull": false + }, + "blocked_reason": { + "name": "blocked_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false + }, + "client_ip": { + "name": "client_ip", + "type": "varchar(45)", + "primaryKey": false, + "notNull": false + }, + "messages_count": { + "name": "messages_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "cache_compatibility_key": { + "name": "cache_compatibility_key", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "cache_score_eligible": { + "name": "cache_score_eligible", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "cache_score_excluded_reason": { + "name": "cache_score_excluded_reason", + "type": "varchar(32)", + "primaryKey": false, + "notNull": false + }, + "theoretical_cache_tokens": { + "name": "theoretical_cache_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "cache_ttl_bucket": { + "name": "cache_ttl_bucket", + "type": "varchar(10)", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "idx_message_request_user_date_cost": { + "name": "idx_message_request_user_date_cost", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "cost_usd", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_user_created_at_cost_stats": { + "name": "idx_message_request_user_created_at_cost_stats", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "cost_usd", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND (\"message_request\".\"blocked_by\" IS NULL OR \"message_request\".\"blocked_by\" <> 'warmup')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_user_query": { + "name": "idx_message_request_user_query", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_provider_created_at_active": { + "name": "idx_message_request_provider_created_at_active", + "columns": [ + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND (\"message_request\".\"blocked_by\" IS NULL OR \"message_request\".\"blocked_by\" <> 'warmup')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_provider_created_at_finalized_active": { + "name": "idx_message_request_provider_created_at_finalized_active", + "columns": [ + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND \"message_request\".\"status_code\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_proxy_status_active": { + "name": "idx_message_request_proxy_status_active", + "columns": [ + { + "expression": "\"created_at\" DESC NULLS LAST", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND \"message_request\".\"is_replay\" = false AND \"message_request\".\"status_code\" IS NULL AND (\"message_request\".\"blocked_by\" IS NULL OR \"message_request\".\"blocked_by\" <> 'warmup')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_proxy_status_latest": { + "name": "idx_message_request_proxy_status_latest", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "\"updated_at\" DESC NULLS LAST", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND \"message_request\".\"is_replay\" = false AND \"message_request\".\"status_code\" IS NOT NULL AND (\"message_request\".\"blocked_by\" IS NULL OR \"message_request\".\"blocked_by\" <> 'warmup')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_session_id": { + "name": "idx_message_request_session_id", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_session_id_prefix": { + "name": "idx_message_request_session_id_prefix", + "columns": [ + { + "expression": "\"session_id\" varchar_pattern_ops", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND (\"message_request\".\"blocked_by\" IS NULL OR \"message_request\".\"blocked_by\" <> 'warmup')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_session_seq": { + "name": "idx_message_request_session_seq", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "request_sequence", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_session_identity_created_at": { + "name": "idx_message_request_session_identity_created_at", + "columns": [ + { + "expression": "COALESCE(\"session_identity\", \"session_id\")", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "\"created_at\" DESC NULLS LAST", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_endpoint": { + "name": "idx_message_request_endpoint", + "columns": [ + { + "expression": "endpoint", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_blocked_by": { + "name": "idx_message_request_blocked_by", + "columns": [ + { + "expression": "blocked_by", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_provider_id": { + "name": "idx_message_request_provider_id", + "columns": [ + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_user_id": { + "name": "idx_message_request_user_id", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_key": { + "name": "idx_message_request_key", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_key_created_at_id": { + "name": "idx_message_request_key_created_at_id", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_key_model_active": { + "name": "idx_message_request_key_model_active", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "model", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND \"message_request\".\"model\" IS NOT NULL AND (\"message_request\".\"blocked_by\" IS NULL OR \"message_request\".\"blocked_by\" <> 'warmup')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_key_endpoint_active": { + "name": "idx_message_request_key_endpoint_active", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "endpoint", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND \"message_request\".\"endpoint\" IS NOT NULL AND (\"message_request\".\"blocked_by\" IS NULL OR \"message_request\".\"blocked_by\" <> 'warmup')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_created_at_id_active": { + "name": "idx_message_request_created_at_id_active", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_model_active": { + "name": "idx_message_request_model_active", + "columns": [ + { + "expression": "model", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND \"message_request\".\"model\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_status_code_active": { + "name": "idx_message_request_status_code_active", + "columns": [ + { + "expression": "status_code", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND \"message_request\".\"status_code\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_created_at": { + "name": "idx_message_request_created_at", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_deleted_at": { + "name": "idx_message_request_deleted_at", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_key_last_active": { + "name": "idx_message_request_key_last_active", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND (\"message_request\".\"blocked_by\" IS NULL OR \"message_request\".\"blocked_by\" <> 'warmup')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_key_cost_active": { + "name": "idx_message_request_key_cost_active", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "cost_usd", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND (\"message_request\".\"blocked_by\" IS NULL OR \"message_request\".\"blocked_by\" <> 'warmup')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_session_user_info": { + "name": "idx_message_request_session_user_info", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_client_ip_created_at": { + "name": "idx_message_request_client_ip_created_at", + "columns": [ + { + "expression": "client_ip", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND \"message_request\".\"client_ip\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.model_prices": { + "name": "model_prices", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "model_name": { + "name": "model_name", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "price_data": { + "name": "price_data", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'cloud'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "idx_model_prices_latest": { + "name": "idx_model_prices_latest", + "columns": [ + { + "expression": "model_name", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_model_prices_model_name": { + "name": "idx_model_prices_model_name", + "columns": [ + { + "expression": "model_name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_model_prices_created_at": { + "name": "idx_model_prices_created_at", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_model_prices_source": { + "name": "idx_model_prices_source", + "columns": [ + { + "expression": "source", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_model_prices_vendor": { + "name": "idx_model_prices_vendor", + "columns": [ + { + "expression": "((\"price_data\" ->> 'vendor'))", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_model_prices_aliases": { + "name": "idx_model_prices_aliases", + "columns": [ + { + "expression": "((\"price_data\" -> 'aliases'))", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.notification_settings": { + "name": "notification_settings", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "use_legacy_mode": { + "name": "use_legacy_mode", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "circuit_breaker_enabled": { + "name": "circuit_breaker_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "circuit_breaker_webhook": { + "name": "circuit_breaker_webhook", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false + }, + "daily_leaderboard_enabled": { + "name": "daily_leaderboard_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "daily_leaderboard_webhook": { + "name": "daily_leaderboard_webhook", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false + }, + "daily_leaderboard_time": { + "name": "daily_leaderboard_time", + "type": "varchar(10)", + "primaryKey": false, + "notNull": false, + "default": "'09:00'" + }, + "daily_leaderboard_top_n": { + "name": "daily_leaderboard_top_n", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 5 + }, + "cost_alert_enabled": { + "name": "cost_alert_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "cost_alert_webhook": { + "name": "cost_alert_webhook", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false + }, + "cost_alert_threshold": { + "name": "cost_alert_threshold", + "type": "numeric(5, 2)", + "primaryKey": false, + "notNull": false, + "default": "'0.80'" + }, + "cost_alert_check_interval": { + "name": "cost_alert_check_interval", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 60 + }, + "cache_hit_rate_alert_enabled": { + "name": "cache_hit_rate_alert_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "cache_hit_rate_alert_webhook": { + "name": "cache_hit_rate_alert_webhook", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false + }, + "cache_hit_rate_alert_window_mode": { + "name": "cache_hit_rate_alert_window_mode", + "type": "varchar(10)", + "primaryKey": false, + "notNull": false, + "default": "'auto'" + }, + "cache_hit_rate_alert_check_interval": { + "name": "cache_hit_rate_alert_check_interval", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 5 + }, + "cache_hit_rate_alert_historical_lookback_days": { + "name": "cache_hit_rate_alert_historical_lookback_days", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 7 + }, + "cache_hit_rate_alert_min_eligible_requests": { + "name": "cache_hit_rate_alert_min_eligible_requests", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 20 + }, + "cache_hit_rate_alert_min_eligible_tokens": { + "name": "cache_hit_rate_alert_min_eligible_tokens", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "cache_hit_rate_alert_abs_min": { + "name": "cache_hit_rate_alert_abs_min", + "type": "numeric(5, 4)", + "primaryKey": false, + "notNull": false, + "default": "'0.05'" + }, + "cache_hit_rate_alert_drop_rel": { + "name": "cache_hit_rate_alert_drop_rel", + "type": "numeric(5, 4)", + "primaryKey": false, + "notNull": false, + "default": "'0.3'" + }, + "cache_hit_rate_alert_drop_abs": { + "name": "cache_hit_rate_alert_drop_abs", + "type": "numeric(5, 4)", + "primaryKey": false, + "notNull": false, + "default": "'0.1'" + }, + "cache_hit_rate_alert_cooldown_minutes": { + "name": "cache_hit_rate_alert_cooldown_minutes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 30 + }, + "cache_hit_rate_alert_top_n": { + "name": "cache_hit_rate_alert_top_n", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 10 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.notification_target_bindings": { + "name": "notification_target_bindings", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "notification_type": { + "name": "notification_type", + "type": "notification_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "target_id": { + "name": "target_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "is_enabled": { + "name": "is_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "schedule_cron": { + "name": "schedule_cron", + "type": "varchar(100)", + "primaryKey": false, + "notNull": false + }, + "schedule_timezone": { + "name": "schedule_timezone", + "type": "varchar(50)", + "primaryKey": false, + "notNull": false + }, + "template_override": { + "name": "template_override", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "unique_notification_target_binding": { + "name": "unique_notification_target_binding", + "columns": [ + { + "expression": "notification_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_notification_bindings_type": { + "name": "idx_notification_bindings_type", + "columns": [ + { + "expression": "notification_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_notification_bindings_target": { + "name": "idx_notification_bindings_target", + "columns": [ + { + "expression": "target_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "notification_target_bindings_target_id_webhook_targets_id_fk": { + "name": "notification_target_bindings_target_id_webhook_targets_id_fk", + "tableFrom": "notification_target_bindings", + "tableTo": "webhook_targets", + "columnsFrom": [ + "target_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.provider_batch_apply_operations": { + "name": "provider_batch_apply_operations", + "schema": "", + "columns": { + "claim_key": { + "name": "claim_key", + "type": "varchar(256)", + "primaryKey": true, + "notNull": true + }, + "preview_token": { + "name": "preview_token", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "payload_fingerprint": { + "name": "payload_fingerprint", + "type": "varchar(128)", + "primaryKey": false, + "notNull": true + }, + "operation_id": { + "name": "operation_id", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "undo_token": { + "name": "undo_token", + "type": "varchar(256)", + "primaryKey": false, + "notNull": true + }, + "undo_expires_at": { + "name": "undo_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "undo_consumed_at": { + "name": "undo_consumed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true + }, + "result": { + "name": "result", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "uniq_provider_batch_apply_operations_preview_token": { + "name": "uniq_provider_batch_apply_operations_preview_token", + "columns": [ + { + "expression": "preview_token", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "uniq_provider_batch_apply_operations_operation_id": { + "name": "uniq_provider_batch_apply_operations_operation_id", + "columns": [ + { + "expression": "operation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "uniq_provider_batch_apply_operations_undo_token": { + "name": "uniq_provider_batch_apply_operations_undo_token", + "columns": [ + { + "expression": "undo_token", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_provider_batch_apply_operations_expires_at": { + "name": "idx_provider_batch_apply_operations_expires_at", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.provider_cache_effectiveness": { + "name": "provider_cache_effectiveness", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "model": { + "name": "model", + "type": "varchar(128)", + "primaryKey": false, + "notNull": true + }, + "cache_ttl_bucket": { + "name": "cache_ttl_bucket", + "type": "varchar(10)", + "primaryKey": false, + "notNull": true + }, + "window_start": { + "name": "window_start", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "window_end": { + "name": "window_end", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "sample_count": { + "name": "sample_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "eligible_count": { + "name": "eligible_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "theoretical_cache_tokens": { + "name": "theoretical_cache_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "observed_cache_read_tokens": { + "name": "observed_cache_read_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "raw_effectiveness_bp": { + "name": "raw_effectiveness_bp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "confidence_bp": { + "name": "confidence_bp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "effectiveness_bp": { + "name": "effectiveness_bp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "idx_provider_cache_effectiveness_window": { + "name": "idx_provider_cache_effectiveness_window", + "columns": [ + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "model", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "window_start", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.provider_endpoint_probe_logs": { + "name": "provider_endpoint_probe_logs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "endpoint_id": { + "name": "endpoint_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'scheduled'" + }, + "ok": { + "name": "ok", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "status_code": { + "name": "status_code", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "latency_ms": { + "name": "latency_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "error_type": { + "name": "error_type", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "idx_provider_endpoint_probe_logs_endpoint_created_at": { + "name": "idx_provider_endpoint_probe_logs_endpoint_created_at", + "columns": [ + { + "expression": "endpoint_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_provider_endpoint_probe_logs_created_at": { + "name": "idx_provider_endpoint_probe_logs_created_at", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "provider_endpoint_probe_logs_endpoint_id_provider_endpoints_id_fk": { + "name": "provider_endpoint_probe_logs_endpoint_id_provider_endpoints_id_fk", + "tableFrom": "provider_endpoint_probe_logs", + "tableTo": "provider_endpoints", + "columnsFrom": [ + "endpoint_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.provider_endpoints": { + "name": "provider_endpoints", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "vendor_id": { + "name": "vendor_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "provider_type": { + "name": "provider_type", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'claude'" + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "label": { + "name": "label", + "type": "varchar(200)", + "primaryKey": false, + "notNull": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "is_enabled": { + "name": "is_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "last_probed_at": { + "name": "last_probed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_probe_ok": { + "name": "last_probe_ok", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "last_probe_status_code": { + "name": "last_probe_status_code", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "last_probe_latency_ms": { + "name": "last_probe_latency_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "last_probe_error_type": { + "name": "last_probe_error_type", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "last_probe_error_message": { + "name": "last_probe_error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "uniq_provider_endpoints_vendor_type_url": { + "name": "uniq_provider_endpoints_vendor_type_url", + "columns": [ + { + "expression": "vendor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "url", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"provider_endpoints\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_provider_endpoints_vendor_type": { + "name": "idx_provider_endpoints_vendor_type", + "columns": [ + { + "expression": "vendor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"provider_endpoints\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_provider_endpoints_enabled": { + "name": "idx_provider_endpoints_enabled", + "columns": [ + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "vendor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"provider_endpoints\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_provider_endpoints_pick_enabled": { + "name": "idx_provider_endpoints_pick_enabled", + "columns": [ + { + "expression": "vendor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "sort_order", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"provider_endpoints\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_provider_endpoints_created_at": { + "name": "idx_provider_endpoints_created_at", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_provider_endpoints_deleted_at": { + "name": "idx_provider_endpoints_deleted_at", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "provider_endpoints_vendor_id_provider_vendors_id_fk": { + "name": "provider_endpoints_vendor_id_provider_vendors_id_fk", + "tableFrom": "provider_endpoints", + "tableTo": "provider_vendors", + "columnsFrom": [ + "vendor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.provider_groups": { + "name": "provider_groups", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(200)", + "primaryKey": false, + "notNull": true + }, + "cost_multiplier": { + "name": "cost_multiplier", + "type": "numeric(10, 4)", + "primaryKey": false, + "notNull": true, + "default": "'1.0'" + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "provider_groups_name_unique": { + "name": "provider_groups_name_unique", + "nullsNotDistinct": false, + "columns": [ + "name" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.provider_vendors": { + "name": "provider_vendors", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "website_domain": { + "name": "website_domain", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "varchar(200)", + "primaryKey": false, + "notNull": false + }, + "website_url": { + "name": "website_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "favicon_url": { + "name": "favicon_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "uniq_provider_vendors_website_domain": { + "name": "uniq_provider_vendors_website_domain", + "columns": [ + { + "expression": "website_domain", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_provider_vendors_created_at": { + "name": "idx_provider_vendors_created_at", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.providers": { + "name": "providers", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "url": { + "name": "url", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "provider_vendor_id": { + "name": "provider_vendor_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "is_enabled": { + "name": "is_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "weight": { + "name": "weight", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "priority": { + "name": "priority", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "group_priorities": { + "name": "group_priorities", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'null'::jsonb" + }, + "cost_multiplier": { + "name": "cost_multiplier", + "type": "numeric(10, 4)", + "primaryKey": false, + "notNull": false, + "default": "'1.0'" + }, + "group_tag": { + "name": "group_tag", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "provider_type": { + "name": "provider_type", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'claude'" + }, + "preserve_client_ip": { + "name": "preserve_client_ip", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "disable_session_reuse": { + "name": "disable_session_reuse", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "model_redirects": { + "name": "model_redirects", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "allowed_models": { + "name": "allowed_models", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'null'::jsonb" + }, + "allowed_clients": { + "name": "allowed_clients", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "blocked_clients": { + "name": "blocked_clients", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "active_time_start": { + "name": "active_time_start", + "type": "varchar(5)", + "primaryKey": false, + "notNull": false + }, + "active_time_end": { + "name": "active_time_end", + "type": "varchar(5)", + "primaryKey": false, + "notNull": false + }, + "codex_instructions_strategy": { + "name": "codex_instructions_strategy", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false, + "default": "'auto'" + }, + "mcp_passthrough_type": { + "name": "mcp_passthrough_type", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'none'" + }, + "mcp_passthrough_url": { + "name": "mcp_passthrough_url", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false + }, + "limit_5h_usd": { + "name": "limit_5h_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "limit_5h_reset_mode": { + "name": "limit_5h_reset_mode", + "type": "daily_reset_mode", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'rolling'" + }, + "limit_daily_usd": { + "name": "limit_daily_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "daily_reset_mode": { + "name": "daily_reset_mode", + "type": "daily_reset_mode", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'fixed'" + }, + "daily_reset_time": { + "name": "daily_reset_time", + "type": "varchar(5)", + "primaryKey": false, + "notNull": true, + "default": "'00:00'" + }, + "limit_weekly_usd": { + "name": "limit_weekly_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "limit_monthly_usd": { + "name": "limit_monthly_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "limit_total_usd": { + "name": "limit_total_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "total_cost_reset_at": { + "name": "total_cost_reset_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "limit_concurrent_sessions": { + "name": "limit_concurrent_sessions", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "max_retry_attempts": { + "name": "max_retry_attempts", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "circuit_breaker_failure_threshold": { + "name": "circuit_breaker_failure_threshold", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 5 + }, + "circuit_breaker_open_duration": { + "name": "circuit_breaker_open_duration", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 1800000 + }, + "circuit_breaker_half_open_success_threshold": { + "name": "circuit_breaker_half_open_success_threshold", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 2 + }, + "proxy_url": { + "name": "proxy_url", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false + }, + "proxy_fallback_to_direct": { + "name": "proxy_fallback_to_direct", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "custom_headers": { + "name": "custom_headers", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "first_byte_timeout_streaming_ms": { + "name": "first_byte_timeout_streaming_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "streaming_idle_timeout_ms": { + "name": "streaming_idle_timeout_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "request_timeout_non_streaming_ms": { + "name": "request_timeout_non_streaming_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "website_url": { + "name": "website_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "favicon_url": { + "name": "favicon_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cache_ttl_preference": { + "name": "cache_ttl_preference", + "type": "varchar(10)", + "primaryKey": false, + "notNull": false + }, + "swap_cache_ttl_billing": { + "name": "swap_cache_ttl_billing", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "context_1m_preference": { + "name": "context_1m_preference", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "codex_reasoning_effort_preference": { + "name": "codex_reasoning_effort_preference", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "codex_reasoning_summary_preference": { + "name": "codex_reasoning_summary_preference", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "codex_text_verbosity_preference": { + "name": "codex_text_verbosity_preference", + "type": "varchar(10)", + "primaryKey": false, + "notNull": false + }, + "codex_parallel_tool_calls_preference": { + "name": "codex_parallel_tool_calls_preference", + "type": "varchar(10)", + "primaryKey": false, + "notNull": false + }, + "codex_image_generation_preference": { + "name": "codex_image_generation_preference", + "type": "varchar(10)", + "primaryKey": false, + "notNull": false + }, + "codex_service_tier_preference": { + "name": "codex_service_tier_preference", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "anthropic_max_tokens_preference": { + "name": "anthropic_max_tokens_preference", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "anthropic_thinking_budget_preference": { + "name": "anthropic_thinking_budget_preference", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "anthropic_adaptive_thinking": { + "name": "anthropic_adaptive_thinking", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'null'::jsonb" + }, + "gemini_google_search_preference": { + "name": "gemini_google_search_preference", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "tpm": { + "name": "tpm", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "rpm": { + "name": "rpm", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "rpd": { + "name": "rpd", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "cc": { + "name": "cc", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "idx_providers_enabled_priority": { + "name": "idx_providers_enabled_priority", + "columns": [ + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "priority", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "weight", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"providers\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_providers_group": { + "name": "idx_providers_group", + "columns": [ + { + "expression": "group_tag", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"providers\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_providers_vendor_type_url_active": { + "name": "idx_providers_vendor_type_url_active", + "columns": [ + { + "expression": "provider_vendor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "url", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"providers\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_providers_created_at": { + "name": "idx_providers_created_at", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_providers_deleted_at": { + "name": "idx_providers_deleted_at", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_providers_vendor_type": { + "name": "idx_providers_vendor_type", + "columns": [ + { + "expression": "provider_vendor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"providers\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_providers_enabled_vendor_type": { + "name": "idx_providers_enabled_vendor_type", + "columns": [ + { + "expression": "provider_vendor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"providers\".\"deleted_at\" IS NULL AND \"providers\".\"is_enabled\" = true AND \"providers\".\"provider_vendor_id\" IS NOT NULL AND \"providers\".\"provider_vendor_id\" > 0", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "providers_provider_vendor_id_provider_vendors_id_fk": { + "name": "providers_provider_vendor_id_provider_vendors_id_fk", + "tableFrom": "providers", + "tableTo": "provider_vendors", + "columnsFrom": [ + "provider_vendor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.replay_payloads": { + "name": "replay_payloads", + "schema": "", + "columns": { + "replay_id": { + "name": "replay_id", + "type": "varchar(64)", + "primaryKey": true, + "notNull": true + }, + "verifier": { + "name": "verifier", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true + }, + "scope_tag": { + "name": "scope_tag", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true + }, + "key_id": { + "name": "key_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "format": { + "name": "format", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true + }, + "model": { + "name": "model", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false + }, + "status_code": { + "name": "status_code", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "headers_json": { + "name": "headers_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "payload": { + "name": "payload", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "byte_size": { + "name": "byte_size", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "source_message_request_id": { + "name": "source_message_request_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "idx_replay_payloads_key_id": { + "name": "idx_replay_payloads_key_id", + "columns": [ + { + "expression": "key_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_replay_payloads_expires_at": { + "name": "idx_replay_payloads_expires_at", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.request_filters": { + "name": "request_filters", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(100)", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true + }, + "action": { + "name": "action", + "type": "varchar(30)", + "primaryKey": false, + "notNull": true + }, + "match_type": { + "name": "match_type", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "target": { + "name": "target", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "replacement": { + "name": "replacement", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "priority": { + "name": "priority", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "is_enabled": { + "name": "is_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "binding_type": { + "name": "binding_type", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'global'" + }, + "provider_ids": { + "name": "provider_ids", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "group_tags": { + "name": "group_tags", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "rule_mode": { + "name": "rule_mode", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'simple'" + }, + "execution_phase": { + "name": "execution_phase", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'guard'" + }, + "operations": { + "name": "operations", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "idx_request_filters_enabled": { + "name": "idx_request_filters_enabled", + "columns": [ + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "priority", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_request_filters_scope": { + "name": "idx_request_filters_scope", + "columns": [ + { + "expression": "scope", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_request_filters_action": { + "name": "idx_request_filters_action", + "columns": [ + { + "expression": "action", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_request_filters_binding": { + "name": "idx_request_filters_binding", + "columns": [ + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "binding_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_request_filters_phase": { + "name": "idx_request_filters_phase", + "columns": [ + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "execution_phase", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sensitive_words": { + "name": "sensitive_words", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "word": { + "name": "word", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "match_type": { + "name": "match_type", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'contains'" + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_enabled": { + "name": "is_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "idx_sensitive_words_enabled": { + "name": "idx_sensitive_words_enabled", + "columns": [ + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "match_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_sensitive_words_created_at": { + "name": "idx_sensitive_words_created_at", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.system_settings": { + "name": "system_settings", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "site_title": { + "name": "site_title", + "type": "varchar(128)", + "primaryKey": false, + "notNull": true, + "default": "'CC Hub'" + }, + "allow_global_usage_view": { + "name": "allow_global_usage_view", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "currency_display": { + "name": "currency_display", + "type": "varchar(10)", + "primaryKey": false, + "notNull": true, + "default": "'USD'" + }, + "billing_model_source": { + "name": "billing_model_source", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'original'" + }, + "codex_priority_billing_source": { + "name": "codex_priority_billing_source", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'requested'" + }, + "bill_non_successful_requests": { + "name": "bill_non_successful_requests", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "bill_hedge_losers": { + "name": "bill_hedge_losers", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "discovery_enabled": { + "name": "discovery_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "discovery_concurrency": { + "name": "discovery_concurrency", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 2 + }, + "max_discovery_rounds": { + "name": "max_discovery_rounds", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 2 + }, + "discovery_sla_ms": { + "name": "discovery_sla_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 10000 + }, + "sticky_sla_ms": { + "name": "sticky_sla_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 20000 + }, + "racing_total_timeout_ms": { + "name": "racing_total_timeout_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 60000 + }, + "sticky_timeout_cooldown_ms": { + "name": "sticky_timeout_cooldown_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 300000 + }, + "timezone": { + "name": "timezone", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "enable_auto_cleanup": { + "name": "enable_auto_cleanup", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "cleanup_retention_days": { + "name": "cleanup_retention_days", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 30 + }, + "cleanup_schedule": { + "name": "cleanup_schedule", + "type": "varchar(50)", + "primaryKey": false, + "notNull": false, + "default": "'0 2 * * *'" + }, + "cleanup_batch_size": { + "name": "cleanup_batch_size", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 10000 + }, + "enable_client_version_check": { + "name": "enable_client_version_check", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "verbose_provider_error": { + "name": "verbose_provider_error", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "pass_through_upstream_error_message": { + "name": "pass_through_upstream_error_message", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "enable_http2": { + "name": "enable_http2", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "enable_openai_responses_websocket": { + "name": "enable_openai_responses_websocket", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "enable_high_concurrency_mode": { + "name": "enable_high_concurrency_mode", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "intercept_anthropic_warmup_requests": { + "name": "intercept_anthropic_warmup_requests", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "enable_thinking_signature_rectifier": { + "name": "enable_thinking_signature_rectifier", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "enable_thinking_budget_rectifier": { + "name": "enable_thinking_budget_rectifier", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "enable_thinking_effort_conflict_rectifier": { + "name": "enable_thinking_effort_conflict_rectifier", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "enable_gemini_function_id_rectifier": { + "name": "enable_gemini_function_id_rectifier", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "enable_billing_header_rectifier": { + "name": "enable_billing_header_rectifier", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "enable_response_input_rectifier": { + "name": "enable_response_input_rectifier", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "allow_non_conversation_endpoint_provider_fallback": { + "name": "allow_non_conversation_endpoint_provider_fallback", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "fake_streaming_whitelist": { + "name": "fake_streaming_whitelist", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "enable_codex_session_id_completion": { + "name": "enable_codex_session_id_completion", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "enable_claude_metadata_user_id_injection": { + "name": "enable_claude_metadata_user_id_injection", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "enable_response_fixer": { + "name": "enable_response_fixer", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "response_fixer_config": { + "name": "response_fixer_config", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'{\"fixTruncatedJson\":true,\"fixSseFormat\":true,\"fixEncoding\":true,\"maxJsonDepth\":200,\"maxFixSize\":1048576}'::jsonb" + }, + "quota_db_refresh_interval_seconds": { + "name": "quota_db_refresh_interval_seconds", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 10 + }, + "quota_lease_percent_5h": { + "name": "quota_lease_percent_5h", + "type": "numeric(5, 4)", + "primaryKey": false, + "notNull": false, + "default": "'0.05'" + }, + "quota_lease_percent_daily": { + "name": "quota_lease_percent_daily", + "type": "numeric(5, 4)", + "primaryKey": false, + "notNull": false, + "default": "'0.05'" + }, + "quota_lease_percent_weekly": { + "name": "quota_lease_percent_weekly", + "type": "numeric(5, 4)", + "primaryKey": false, + "notNull": false, + "default": "'0.05'" + }, + "quota_lease_percent_monthly": { + "name": "quota_lease_percent_monthly", + "type": "numeric(5, 4)", + "primaryKey": false, + "notNull": false, + "default": "'0.05'" + }, + "quota_lease_cap_usd": { + "name": "quota_lease_cap_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "ip_extraction_config": { + "name": "ip_extraction_config", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "ip_geo_lookup_enabled": { + "name": "ip_geo_lookup_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "public_status_window_hours": { + "name": "public_status_window_hours", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 24 + }, + "public_status_aggregation_interval_minutes": { + "name": "public_status_aggregation_interval_minutes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 5 + }, + "stream_gate_mode": { + "name": "stream_gate_mode", + "type": "varchar(10)", + "primaryKey": false, + "notNull": true, + "default": "'enforce'" + }, + "affinity_ignore_client_session_id": { + "name": "affinity_ignore_client_session_id", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "replay_enabled": { + "name": "replay_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "replay_cache_ttl_minutes": { + "name": "replay_cache_ttl_minutes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 30 + }, + "cache_effectiveness_enabled": { + "name": "cache_effectiveness_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.usage_ledger": { + "name": "usage_ledger", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "request_id": { + "name": "request_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "final_provider_id": { + "name": "final_provider_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "model": { + "name": "model", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false + }, + "original_model": { + "name": "original_model", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false + }, + "actual_response_model": { + "name": "actual_response_model", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false + }, + "endpoint": { + "name": "endpoint", + "type": "varchar(256)", + "primaryKey": false, + "notNull": false + }, + "api_type": { + "name": "api_type", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "session_id": { + "name": "session_id", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "session_identity": { + "name": "session_identity", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "session_identity_kind": { + "name": "session_identity_kind", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "affinity_scope_tag": { + "name": "affinity_scope_tag", + "type": "varchar(16)", + "primaryKey": false, + "notNull": false + }, + "affinity_fingerprint": { + "name": "affinity_fingerprint", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "affinity_fingerprint_chain": { + "name": "affinity_fingerprint_chain", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "is_replay": { + "name": "is_replay", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "replay_source_request_id": { + "name": "replay_source_request_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "status_code": { + "name": "status_code", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "is_success": { + "name": "is_success", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "success_rate_outcome": { + "name": "success_rate_outcome", + "type": "varchar(16)", + "primaryKey": false, + "notNull": false + }, + "blocked_by": { + "name": "blocked_by", + "type": "varchar(50)", + "primaryKey": false, + "notNull": false + }, + "cost_usd": { + "name": "cost_usd", + "type": "numeric(21, 15)", + "primaryKey": false, + "notNull": false, + "default": "'0'" + }, + "cost_multiplier": { + "name": "cost_multiplier", + "type": "numeric(10, 4)", + "primaryKey": false, + "notNull": false + }, + "group_cost_multiplier": { + "name": "group_cost_multiplier", + "type": "numeric(10, 4)", + "primaryKey": false, + "notNull": false + }, + "input_tokens": { + "name": "input_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "output_tokens": { + "name": "output_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "cache_creation_input_tokens": { + "name": "cache_creation_input_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "cache_read_input_tokens": { + "name": "cache_read_input_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "cache_creation_5m_input_tokens": { + "name": "cache_creation_5m_input_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "cache_creation_1h_input_tokens": { + "name": "cache_creation_1h_input_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "cache_ttl_applied": { + "name": "cache_ttl_applied", + "type": "varchar(10)", + "primaryKey": false, + "notNull": false + }, + "context_1m_applied": { + "name": "context_1m_applied", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "swap_cache_ttl_applied": { + "name": "swap_cache_ttl_applied", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "ttfb_ms": { + "name": "ttfb_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "first_byte_ms": { + "name": "first_byte_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "client_ip": { + "name": "client_ip", + "type": "varchar(45)", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "idx_usage_ledger_request_id": { + "name": "idx_usage_ledger_request_id", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_user_created_at": { + "name": "idx_usage_ledger_user_created_at", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_ledger\".\"blocked_by\" IS NULL AND \"usage_ledger\".\"is_replay\" = false", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_user_id_reset": { + "name": "idx_usage_ledger_user_id_reset", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_key_created_at": { + "name": "idx_usage_ledger_key_created_at", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_ledger\".\"blocked_by\" IS NULL AND \"usage_ledger\".\"is_replay\" = false", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_provider_created_at": { + "name": "idx_usage_ledger_provider_created_at", + "columns": [ + { + "expression": "final_provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_ledger\".\"blocked_by\" IS NULL AND \"usage_ledger\".\"is_replay\" = false", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_created_at_minute": { + "name": "idx_usage_ledger_created_at_minute", + "columns": [ + { + "expression": "date_trunc('minute', \"created_at\" AT TIME ZONE 'UTC')", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_created_at_desc_id": { + "name": "idx_usage_ledger_created_at_desc_id", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_session_id": { + "name": "idx_usage_ledger_session_id", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_ledger\".\"session_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_session_identity_created_at": { + "name": "idx_usage_ledger_session_identity_created_at", + "columns": [ + { + "expression": "COALESCE(\"session_identity\", \"session_id\")", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "\"created_at\" DESC NULLS LAST", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_ledger\".\"blocked_by\" IS NULL AND \"usage_ledger\".\"is_replay\" = false", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_session_identity": { + "name": "idx_usage_ledger_session_identity", + "columns": [ + { + "expression": "COALESCE(\"session_identity\", \"session_id\")", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_model": { + "name": "idx_usage_ledger_model", + "columns": [ + { + "expression": "model", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_ledger\".\"model\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_key_cost": { + "name": "idx_usage_ledger_key_cost", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "cost_usd", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "endpoint", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_ledger\".\"blocked_by\" IS NULL AND \"usage_ledger\".\"is_replay\" = false", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_user_cost_cover": { + "name": "idx_usage_ledger_user_cost_cover", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "cost_usd", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "endpoint", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_ledger\".\"blocked_by\" IS NULL AND \"usage_ledger\".\"is_replay\" = false", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_provider_cost_cover": { + "name": "idx_usage_ledger_provider_cost_cover", + "columns": [ + { + "expression": "final_provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "cost_usd", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "endpoint", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_ledger\".\"blocked_by\" IS NULL AND \"usage_ledger\".\"is_replay\" = false", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_key_created_at_desc_cover": { + "name": "idx_usage_ledger_key_created_at_desc_cover", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "\"created_at\" DESC NULLS LAST", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "final_provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_ledger\".\"blocked_by\" IS NULL AND \"usage_ledger\".\"is_replay\" = false", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.users": { + "name": "users", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "role": { + "name": "role", + "type": "varchar", + "primaryKey": false, + "notNull": false, + "default": "'user'" + }, + "rpm_limit": { + "name": "rpm_limit", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "daily_limit_usd": { + "name": "daily_limit_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "provider_group": { + "name": "provider_group", + "type": "varchar(200)", + "primaryKey": false, + "notNull": false, + "default": "'default'" + }, + "tags": { + "name": "tags", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'[]'::jsonb" + }, + "limit_5h_usd": { + "name": "limit_5h_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "limit_5h_reset_mode": { + "name": "limit_5h_reset_mode", + "type": "daily_reset_mode", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'rolling'" + }, + "limit_weekly_usd": { + "name": "limit_weekly_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "limit_monthly_usd": { + "name": "limit_monthly_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "limit_total_usd": { + "name": "limit_total_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "cost_reset_at": { + "name": "cost_reset_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "limit_5h_cost_reset_at": { + "name": "limit_5h_cost_reset_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "limit_concurrent_sessions": { + "name": "limit_concurrent_sessions", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "daily_reset_mode": { + "name": "daily_reset_mode", + "type": "daily_reset_mode", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'fixed'" + }, + "daily_reset_time": { + "name": "daily_reset_time", + "type": "varchar(5)", + "primaryKey": false, + "notNull": true, + "default": "'00:00'" + }, + "is_enabled": { + "name": "is_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "allowed_clients": { + "name": "allowed_clients", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'[]'::jsonb" + }, + "allowed_models": { + "name": "allowed_models", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'[]'::jsonb" + }, + "blocked_clients": { + "name": "blocked_clients", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "idx_users_active_role_sort": { + "name": "idx_users_active_role_sort", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "role", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"users\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_users_enabled_expires_at": { + "name": "idx_users_enabled_expires_at", + "columns": [ + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"users\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_users_tags_gin": { + "name": "idx_users_tags_gin", + "columns": [ + { + "expression": "tags", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"users\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "gin", + "with": {} + }, + "idx_users_created_at": { + "name": "idx_users_created_at", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_users_deleted_at": { + "name": "idx_users_deleted_at", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.webhook_targets": { + "name": "webhook_targets", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(100)", + "primaryKey": false, + "notNull": true + }, + "provider_type": { + "name": "provider_type", + "type": "webhook_provider_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "webhook_url": { + "name": "webhook_url", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": false + }, + "telegram_bot_token": { + "name": "telegram_bot_token", + "type": "varchar(256)", + "primaryKey": false, + "notNull": false + }, + "telegram_chat_id": { + "name": "telegram_chat_id", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "dingtalk_secret": { + "name": "dingtalk_secret", + "type": "varchar(256)", + "primaryKey": false, + "notNull": false + }, + "custom_template": { + "name": "custom_template", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "custom_headers": { + "name": "custom_headers", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "proxy_url": { + "name": "proxy_url", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false + }, + "proxy_fallback_to_direct": { + "name": "proxy_fallback_to_direct", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "is_enabled": { + "name": "is_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "last_test_at": { + "name": "last_test_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_test_result": { + "name": "last_test_result", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.outbox_events": { + "name": "outbox_events", + "schema": "public", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "event_id": { + "name": "event_id", + "type": "uuid", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()" + }, + "event_type": { + "name": "event_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "aggregate_type": { + "name": "aggregate_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "aggregate_id": { + "name": "aggregate_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "occurred_at": { + "name": "occurred_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "published_at": { + "name": "published_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": "0" + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "idx_outbox_events_unpublished": { + "name": "idx_outbox_events_unpublished", + "columns": [ + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {}, + "where": "\"outbox_events\".\"published_at\" IS NULL" + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "outbox_events_event_id_key": { + "name": "outbox_events_event_id_key", + "nullsNotDistinct": false, + "columns": [ + "event_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.outbox_processed": { + "name": "outbox_processed", + "schema": "public", + "columns": { + "event_id": { + "name": "event_id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "processed_at": { + "name": "processed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.proj_applied_requests": { + "name": "proj_applied_requests", + "schema": "public", + "columns": { + "request_id": { + "name": "request_id", + "type": "bigint", + "primaryKey": true, + "notNull": true + }, + "event_id": { + "name": "event_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "applied_at": { + "name": "applied_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.avail_bucket_1m": { + "name": "avail_bucket_1m", + "schema": "public", + "columns": { + "provider_id": { + "name": "provider_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "bucket_start": { + "name": "bucket_start", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "success_cnt": { + "name": "success_cnt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": "0" + }, + "failure_cnt": { + "name": "failure_cnt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": "0" + }, + "excluded_cnt": { + "name": "excluded_cnt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": "0" + }, + "latency_cnt": { + "name": "latency_cnt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": "0" + }, + "latency_sum_ms": { + "name": "latency_sum_ms", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": "0" + }, + "last_request_at": { + "name": "last_request_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "idx_avail_bucket_1m_time": { + "name": "idx_avail_bucket_1m_time", + "columns": [ + { + "expression": "bucket_start", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "avail_bucket_1m_pkey": { + "name": "avail_bucket_1m_pkey", + "columns": [ + "provider_id", + "bucket_start" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.avail_current": { + "name": "avail_current", + "schema": "public", + "columns": { + "provider_id": { + "name": "provider_id", + "type": "integer", + "primaryKey": true, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'unknown'" + }, + "availability": { + "name": "availability", + "type": "double precision", + "primaryKey": false, + "notNull": true, + "default": "0" + }, + "request_count": { + "name": "request_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": "0" + }, + "last_request_at": { + "name": "last_request_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.projection_meta": { + "name": "projection_meta", + "schema": "public", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "value": { + "name": "value", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.daily_reset_mode": { + "name": "daily_reset_mode", + "schema": "public", + "values": [ + "fixed", + "rolling" + ] + }, + "public.notification_type": { + "name": "notification_type", + "schema": "public", + "values": [ + "circuit_breaker", + "daily_leaderboard", + "cost_alert", + "cache_hit_rate_alert" + ] + }, + "public.webhook_provider_type": { + "name": "webhook_provider_type", + "schema": "public", + "values": [ + "wechat", + "feishu", + "dingtalk", + "telegram", + "custom" + ] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index f6b450d14..6e10f0a36 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -841,6 +841,13 @@ "when": 1786038550610, "tag": "0119_tiresome_banshee", "breakpoints": true + }, + { + "idx": 120, + "version": "7", + "when": 1786500000000, + "tag": "0120_availability_projection", + "breakpoints": true } ] -} \ No newline at end of file +} diff --git a/src/drizzle/schema.ts b/src/drizzle/schema.ts index fea464b10..08088d73d 100644 --- a/src/drizzle/schema.ts +++ b/src/drizzle/schema.ts @@ -1425,3 +1425,13 @@ export const messageRequestRelations = relations(messageRequest, ({ one }) => ({ references: [providers.id], }), })); + +// Availability projection tables (outbox + 1m buckets). Source of truth for drizzle-kit. +export { + availBucket1m, + availCurrent, + outboxEvents, + outboxProcessed, + projAppliedRequests, + projectionMeta, +} from "@/lib/availability/projection-tables"; diff --git a/src/instrumentation.ts b/src/instrumentation.ts index a14f0003d..5696092a8 100644 --- a/src/instrumentation.ts +++ b/src/instrumentation.ts @@ -639,6 +639,17 @@ export async function register() { }); } + try { + const { startAvailabilityProjectionWorker } = await import( + "@/lib/availability/projection-worker" + ); + startAvailabilityProjectionWorker(); + } catch (error) { + logger.warn("[Instrumentation] Failed to start availability projection worker", { + error: error instanceof Error ? error.message : String(error), + }); + } + // 初始化端点熔断器（禁用时清理残留状态） try { const { initEndpointCircuitBreaker } = await import("@/lib/endpoint-circuit-breaker"); @@ -808,6 +819,17 @@ export async function register() { }); } + try { + const { startAvailabilityProjectionWorker } = await import( + "@/lib/availability/projection-worker" + ); + startAvailabilityProjectionWorker(); + } catch (error) { + logger.warn("[Instrumentation] Failed to start availability projection worker", { + error: error instanceof Error ? error.message : String(error), + }); + } + // 初始化端点熔断器（禁用时清理残留状态） try { const { initEndpointCircuitBreaker } = await import("@/lib/endpoint-circuit-breaker"); diff --git a/src/lib/availability/availability-service.ts b/src/lib/availability/availability-service.ts index d465cb31d..28cee31e8 100644 --- a/src/lib/availability/availability-service.ts +++ b/src/lib/availability/availability-service.ts @@ -2,11 +2,14 @@ * Provider Availability Aggregation Service * Calculates availability metrics from request logs * Simple two-tier status: success (green) or failure (red) + * + * Read path uses incremental 1-minute projection buckets (avail_bucket_1m). + * message_request is no longer scanned here. */ -import { and, eq, inArray, isNotNull, isNull, type SQLWrapper, sql } from "drizzle-orm"; +import { and, eq, inArray, isNull, sql } from "drizzle-orm"; import { db } from "@/drizzle/db"; -import { messageRequest, providers } from "@/drizzle/schema"; +import { providers } from "@/drizzle/schema"; import type { AvailabilityQueryOptions, AvailabilityQueryResult, @@ -30,31 +33,34 @@ type AggregatedAvailabilityBucketRow = { lastRequestAt: Date | null; }; -type AggregatedCurrentProviderStatusRow = { - providerId: number; - greenCount: number; - redCount: number; - lastRequestAt: Date | null; -}; - export const MIN_BUCKET_SIZE_MINUTES = 0.25; export const MAX_BUCKET_SIZE_MINUTES = 1440; const DEFAULT_MAX_BUCKETS = 100; const AVAILABILITY_SUCCESS_STATUS_CODE_MIN = 200; const AVAILABILITY_SUCCESS_STATUS_CODE_MAX_EXCLUSIVE = 400; -const FINALIZED_REQUEST_OUTCOME_ALIAS = "successRateOutcome" as const; -const FINALIZED_REQUEST_OUTCOME_SQL = sql.raw(`"${FINALIZED_REQUEST_OUTCOME_ALIAS}"`); -const COUNTABLE_REQUEST_OUTCOME_SQL = sql`${FINALIZED_REQUEST_OUTCOME_SQL} IN ('success', 'failure')`; // Keep the hard cap independent from the UI/API default so future default tuning does not silently relax/tighten the guardrail. // It intentionally equals the default today; the separation preserves distinct semantic roles for future tuning. export const MAX_BUCKETS_HARD_LIMIT = 100; -const CURRENT_PROVIDER_STATUS_WINDOW_MINUTES = 15; +/** Shared window for avail_current freshness and getCurrentProviderStatus fallback. */ +export const CURRENT_PROVIDER_STATUS_WINDOW_MINUTES = 15; export const MAX_AVAILABILITY_QUERY_RANGE_DAYS = (MAX_BUCKETS_HARD_LIMIT * MAX_BUCKET_SIZE_MINUTES) / (24 * 60); const MAX_AVAILABILITY_QUERY_RANGE_MS = MAX_BUCKETS_HARD_LIMIT * MAX_BUCKET_SIZE_MINUTES * 60 * 1000; +function floorToUtcMinute(date: Date): Date { + return new Date(Date.UTC( + date.getUTCFullYear(), + date.getUTCMonth(), + date.getUTCDate(), + date.getUTCHours(), + date.getUTCMinutes(), + 0, + 0 + )); +} + export class AvailabilityQueryValidationError extends Error { constructor(message: string) { super(message); @@ -62,27 +68,6 @@ export class AvailabilityQueryValidationError extends Error { } } -/** - * 可用性监控的"已终态"边界收敛为 `status_code IS NOT NULL`。 - * - * 这与部分索引 `idx_message_request_provider_created_at_finalized_active` - * 的谓词 `deleted_at IS NULL AND status_code IS NOT NULL` 对齐，让 - * provider + 时间范围聚合可以直接命中索引，而不是退化为大范围扫描。 - * - * 不复刻 `fn_is_message_request_finalized` 的语义（即使内联）也是有意为之： - * 该函数会把仅有 providerChain / errorMessage 片段但 statusCode 仍为 NULL - * 的"请求中"记录判为终态；放到可用性统计里会被分类函数误算成 failure。 - * 终态记录的成功/失败/排除分类继续由 - * `fn_compute_message_request_success_rate_outcome(...)` 处理。 - * - * 已知限制：若未来出现 status_code 长时间未落库但请求已稳定结束的写路径， - * 这些记录会被排除；届时应引入独立的、SARGable 的 finalized 谓词， - * 而不是放回 PL/pgSQL 函数调用。 - */ -function buildAvailabilityFinalizedCondition() { - return isNotNull(messageRequest.statusCode); -} - function assertValidDate(date: Date, fieldName: string): Date { if (!Number.isFinite(date.getTime())) { throw new AvailabilityQueryValidationError( @@ -97,50 +82,6 @@ function parseAvailabilityDate(value: Date | string, fieldName: string): Date { return assertValidDate(typeof value === "string" ? new Date(value) : value, fieldName); } -function buildTimestampLowerBound( - column: typeof messageRequest.createdAt, - date: Date, - fieldName: string -) { - return sql`${column} >= CAST(${assertValidDate(date, fieldName).toISOString()} AS timestamptz)`; -} - -function buildTimestampUpperBound( - column: typeof messageRequest.createdAt, - date: Date, - fieldName: string -) { - return sql`${column} <= CAST(${assertValidDate(date, fieldName).toISOString()} AS timestamptz)`; -} - -function buildRelativeNowLowerBound(column: typeof messageRequest.createdAt, minutes: number) { - return sql`${column} >= NOW() - (${sql.raw(String(minutes))} * INTERVAL '1 minute')`; -} - -function buildNowUpperBound(column: typeof messageRequest.createdAt) { - return sql`${column} <= NOW()`; -} - -function buildAvailabilityRequestConditions(input: { - providerIds: number[]; - startDate: Date; - endDate?: Date; -}) { - const conditions = [ - inArray(messageRequest.providerId, input.providerIds), - buildTimestampLowerBound(messageRequest.createdAt, input.startDate, "startTime"), - isNull(messageRequest.deletedAt), - eq(messageRequest.isReplay, false), - buildAvailabilityFinalizedCondition(), - ]; - - if (input.endDate) { - conditions.push(buildTimestampUpperBound(messageRequest.createdAt, input.endDate, "endTime")); - } - - return and(...conditions); -} - function toFiniteNumber(value: number | string | null | undefined): number { const parsed = Number(value ?? 0); return Number.isFinite(parsed) ? parsed : 0; @@ -168,28 +109,6 @@ function isAvailabilitySuccessStatusCode(statusCode: number): boolean { ); } -function buildRequestOutcomeSql( - blockedByExpression: SQLWrapper, - statusCodeExpression: SQLWrapper, - errorMessageExpression: SQLWrapper, - providerChainExpression: SQLWrapper -) { - return sql`fn_compute_message_request_success_rate_outcome( - ${blockedByExpression}, - ${statusCodeExpression}, - ${errorMessageExpression}, - ${providerChainExpression} - )`; -} - -function buildAvailabilitySuccessOutcomeCondition(outcomeExpression: SQLWrapper) { - return sql`${outcomeExpression} = 'success'`; -} - -function buildAvailabilityFailureOutcomeCondition(outcomeExpression: SQLWrapper) { - return sql`${outcomeExpression} = 'failure'`; -} - /** * Classify a single finalized request's status * Simple: success (2xx/3xx) = green, failure = red @@ -306,7 +225,9 @@ function validateAvailabilityTimeRange(startDate: Date, endDate: Date): void { } /** - * Query availability data for providers + * Query availability data for providers. + * Read path uses incremental 1-minute projection buckets (avail_bucket_1m). + * message_request is no longer scanned here. */ export async function queryProviderAvailability( options: AvailabilityQueryOptions = {} @@ -333,7 +254,6 @@ export async function queryProviderAvailability( sanitizedMaxBuckets ); const bucketSizeMs = bucketSizeMinutes * 60 * 1000; - const bucketSizeSeconds = bucketSizeMinutes * 60; // Get provider list const providerConditions = [isNull(providers.deletedAt)]; @@ -366,75 +286,59 @@ export async function queryProviderAvailability( } const providerIdList = providerList.map((provider) => provider.id); - const requestConditions = buildAvailabilityRequestConditions({ - providerIds: providerIdList, - startDate, - endDate, - }); - const availabilityAggregationCtes = sql` - finalized_requests AS ( + // Include every 1m bucket that overlaps [startDate, endDate]: floor start, keep end exclusive upper via end. + const rangeStartBucket = floorToUtcMinute(startDate); + const rangeEndBucket = floorToUtcMinute(endDate); + + // Aggregate pre-projected 1-minute buckets into the requested display bucket size. + // p50/p95/p99 currently equal avg (sum/count) until sketch-based percentiles land — + // field names stay for API compatibility; treat them as mean approximations. + const bucketQuery = sql` + WITH provider_bucket_stats AS ( SELECT - ${messageRequest.providerId} AS "providerId", - ${messageRequest.createdAt} AS "createdAt", - ${buildRequestOutcomeSql( - messageRequest.blockedBy, - messageRequest.statusCode, - messageRequest.errorMessage, - messageRequest.providerChain - )} AS ${FINALIZED_REQUEST_OUTCOME_SQL}, - ${messageRequest.durationMs} AS "durationMs", - to_timestamp( - floor(extract(epoch from ${messageRequest.createdAt}) / ${bucketSizeSeconds}) * ${bucketSizeSeconds} - ) AS "bucketStart" - FROM ${messageRequest} - WHERE ${requestConditions} + provider_id AS "providerId", + date_bin( + (${bucketSizeMinutes} * INTERVAL '1 minute'), + bucket_start, + TIMESTAMPTZ '1970-01-01T00:00:00Z' + ) AS "bucketStart", + SUM(success_cnt)::int AS "greenCount", + SUM(failure_cnt)::int AS "redCount", + SUM(latency_cnt)::int AS "latencyCount", + COALESCE(SUM(latency_sum_ms), 0)::double precision AS "latencySumMs", + CASE + WHEN SUM(latency_cnt) > 0 THEN (SUM(latency_sum_ms)::double precision / SUM(latency_cnt)) + ELSE 0 + END AS "avgLatencyMs", + CASE + WHEN SUM(latency_cnt) > 0 THEN (SUM(latency_sum_ms)::double precision / SUM(latency_cnt)) + ELSE 0 + END AS "p50LatencyMs", + CASE + WHEN SUM(latency_cnt) > 0 THEN (SUM(latency_sum_ms)::double precision / SUM(latency_cnt)) + ELSE 0 + END AS "p95LatencyMs", + CASE + WHEN SUM(latency_cnt) > 0 THEN (SUM(latency_sum_ms)::double precision / SUM(latency_cnt)) + ELSE 0 + END AS "p99LatencyMs", + MAX(last_request_at) AS "lastRequestAt" + FROM avail_bucket_1m + WHERE provider_id IN (${sql.join( + providerIdList.map((id) => sql`${id}`), + sql`, ` + )}) + AND bucket_start >= CAST(${rangeStartBucket.toISOString()} AS timestamptz) + AND bucket_start <= CAST(${rangeEndBucket.toISOString()} AS timestamptz) + GROUP BY provider_id, 2 ), - provider_bucket_stats AS ( + limited_provider_bucket_stats AS ( SELECT - "providerId", - "bucketStart", - COUNT(*) FILTER (WHERE ${buildAvailabilitySuccessOutcomeCondition(FINALIZED_REQUEST_OUTCOME_SQL)})::int AS "greenCount", - COUNT(*) FILTER (WHERE ${buildAvailabilityFailureOutcomeCondition(FINALIZED_REQUEST_OUTCOME_SQL)})::int AS "redCount", - COUNT("durationMs") FILTER (WHERE ${COUNTABLE_REQUEST_OUTCOME_SQL})::int AS "latencyCount", - COALESCE( - SUM("durationMs") FILTER (WHERE ${COUNTABLE_REQUEST_OUTCOME_SQL})::double precision, - 0 - ) AS "latencySumMs", - COALESCE( - AVG("durationMs") FILTER (WHERE ${COUNTABLE_REQUEST_OUTCOME_SQL})::double precision, - 0 - ) AS "avgLatencyMs", - COALESCE( - percentile_cont(0.5) WITHIN GROUP (ORDER BY "durationMs"::double precision) - FILTER (WHERE "durationMs" IS NOT NULL AND ${COUNTABLE_REQUEST_OUTCOME_SQL}), - 0 - )::double precision AS "p50LatencyMs", - COALESCE( - percentile_cont(0.95) WITHIN GROUP (ORDER BY "durationMs"::double precision) - FILTER (WHERE "durationMs" IS NOT NULL AND ${COUNTABLE_REQUEST_OUTCOME_SQL}), - 0 - )::double precision AS "p95LatencyMs", - COALESCE( - percentile_cont(0.99) WITHIN GROUP (ORDER BY "durationMs"::double precision) - FILTER (WHERE "durationMs" IS NOT NULL AND ${COUNTABLE_REQUEST_OUTCOME_SQL}), - 0 - )::double precision AS "p99LatencyMs", - MAX("createdAt") AS "lastRequestAt" - FROM finalized_requests - GROUP BY "providerId", "bucketStart" + *, + ROW_NUMBER() OVER (PARTITION BY "providerId" ORDER BY "bucketStart" DESC) AS rn + FROM provider_bucket_stats ) - `; - - const bucketQuery = sql` - WITH - ${availabilityAggregationCtes}, - limited_provider_bucket_stats AS ( - SELECT - *, - ROW_NUMBER() OVER (PARTITION BY "providerId" ORDER BY "bucketStart" DESC) AS rn - FROM provider_bucket_stats - ) SELECT "providerId", "bucketStart", @@ -565,7 +469,7 @@ export async function queryProviderAvailability( } /** - * Get current availability status for all providers (lightweight query) + * Get current availability status for all providers (lightweight, projection table). */ export async function getCurrentProviderStatus(): Promise< Array<{ @@ -590,90 +494,116 @@ export async function getCurrentProviderStatus(): Promise< return []; } - const providerIdList = providerList.map((provider) => provider.id); - const requestConditions = and( - inArray(messageRequest.providerId, providerIdList), - buildRelativeNowLowerBound(messageRequest.createdAt, CURRENT_PROVIDER_STATUS_WINDOW_MINUTES), - buildNowUpperBound(messageRequest.createdAt), - isNull(messageRequest.deletedAt), - eq(messageRequest.isReplay, false), - buildAvailabilityFinalizedCondition() - ); + type CurrentRow = { + providerId: number; + state: string; + availability: number; + requestCount: number; + lastRequestAt: Date | string | null; + updatedAt: Date | string | null; + }; - const aggregateQuery = sql` - SELECT - ${messageRequest.providerId} AS "providerId", - COUNT(*) FILTER (WHERE ${buildAvailabilitySuccessOutcomeCondition( - buildRequestOutcomeSql( - messageRequest.blockedBy, - messageRequest.statusCode, - messageRequest.errorMessage, - messageRequest.providerChain - ) - )})::int AS "greenCount", - COUNT(*) FILTER (WHERE ${buildAvailabilityFailureOutcomeCondition( - buildRequestOutcomeSql( - messageRequest.blockedBy, - messageRequest.statusCode, - messageRequest.errorMessage, - messageRequest.providerChain - ) - )})::int AS "redCount", - MAX(${messageRequest.createdAt}) AS "lastRequestAt" - FROM ${messageRequest} - WHERE ${requestConditions} - GROUP BY ${messageRequest.providerId} - `; + const windowMs = CURRENT_PROVIDER_STATUS_WINDOW_MINUTES * 60 * 1000; + const nowMs = Date.now(); - const aggregateRows = Array.from( - await db.execute(aggregateQuery) - ) as AggregatedCurrentProviderStatusRow[]; - const providerStats = new Map< - number, - { - greenCount: number; - redCount: number; - lastRequestAt: string | null; + const currentRows = await db.execute(sql` + SELECT + c.provider_id AS "providerId", + c.state AS "state", + c.availability AS "availability", + c.request_count AS "requestCount", + c.last_request_at AS "lastRequestAt", + c.updated_at AS "updatedAt" + FROM avail_current c + WHERE c.provider_id IN (${sql.join( + providerList.map((p) => sql`${p.id}`), + sql`, ` + )}) + `); + + const byId = new Map(); + for (const row of Array.from(currentRows as Iterable)) { + const updatedAtMs = getTimeValue(row.updatedAt); + const lastRequestAtMs = getTimeValue(row.lastRequestAt); + const freshAt = Math.max(updatedAtMs, lastRequestAtMs); + // Idle providers must not keep a frozen green/red forever. + if (freshAt <= 0 || nowMs - freshAt > windowMs) { + continue; } - >(); - - for (const provider of providerList) { - providerStats.set(provider.id, { - greenCount: 0, - redCount: 0, - lastRequestAt: null, - }); + byId.set(Number(row.providerId), row); } - for (const row of aggregateRows) { - providerStats.set(row.providerId, { - greenCount: toFiniteNumber(row.greenCount), - redCount: toFiniteNumber(row.redCount), - lastRequestAt: toIsoString(row.lastRequestAt), - }); + const missing = providerList.filter((p) => !byId.has(p.id)).map((p) => p.id); + if (missing.length > 0) { + const fallback = await db.execute(sql` + SELECT + provider_id AS "providerId", + SUM(success_cnt)::int AS "greenCount", + SUM(failure_cnt)::int AS "redCount", + MAX(last_request_at) AS "lastRequestAt" + FROM avail_bucket_1m + WHERE provider_id IN (${sql.join( + missing.map((id) => sql`${id}`), + sql`, ` + )}) + AND bucket_start >= NOW() - (${sql.raw(String(CURRENT_PROVIDER_STATUS_WINDOW_MINUTES))} * INTERVAL '1 minute') + GROUP BY provider_id + `); + + for (const row of Array.from( + fallback as Iterable<{ + providerId: number; + greenCount: number; + redCount: number; + lastRequestAt: Date | string | null; + }> + )) { + const g = toFiniteNumber(row.greenCount); + const r = toFiniteNumber(row.redCount); + const total = g + r; + if (total <= 0) continue; + const availability = calculateAvailabilityScore(g, r); + byId.set(Number(row.providerId), { + providerId: Number(row.providerId), + state: availability >= 0.5 ? "green" : "red", + availability, + requestCount: total, + lastRequestAt: row.lastRequestAt, + updatedAt: row.lastRequestAt, + }); + } } return providerList.map((provider) => { - const stats = providerStats.get(provider.id)!; - const total = stats.greenCount + stats.redCount; - const availability = calculateAvailabilityScore(stats.greenCount, stats.redCount); + const stats = byId.get(provider.id); + if (!stats || toFiniteNumber(stats.requestCount) <= 0) { + return { + providerId: provider.id, + providerName: provider.name, + status: "unknown" as AvailabilityStatus, + availability: 0, + requestCount: 0, + lastRequestAt: null, + }; + } - // IMPORTANT: No data = 'unknown', NOT 'green'! Must be honest. + const rawState = String(stats.state || "unknown"); let status: AvailabilityStatus = "unknown"; - if (total === 0) { - status = "unknown"; // No data - must be honest, don't assume healthy! + if (rawState === "green" || rawState === "red" || rawState === "unknown") { + status = rawState; + } else if (rawState === "yellow") { + status = toFiniteNumber(stats.availability) >= 0.5 ? "green" : "red"; } else { - // Simple: >= 50% success = green, otherwise red - status = availability >= 0.5 ? "green" : "red"; + status = toFiniteNumber(stats.availability) >= 0.5 ? "green" : "red"; } return { providerId: provider.id, providerName: provider.name, status, - availability, - requestCount: total, - lastRequestAt: stats.lastRequestAt, + availability: toFiniteNumber(stats.availability), + requestCount: toFiniteNumber(stats.requestCount), + lastRequestAt: toIsoString(stats.lastRequestAt), }; }); } diff --git a/src/lib/availability/index.ts b/src/lib/availability/index.ts index b795312ee..369dc5bbf 100644 --- a/src/lib/availability/index.ts +++ b/src/lib/availability/index.ts @@ -1,9 +1,10 @@ /** * Provider Availability Module * - * This module provides availability monitoring based on request log data. - * Availability is calculated only from finalized requests that already have a persisted - * `statusCode`. In-flight / intermediate records are excluded upstream. + * Read path aggregates pre-projected 1-minute buckets (avail_bucket_1m / avail_current). + * Write path finalization still relies on message_request.statusCode: a DB trigger enqueues + * outbox events, and the in-process projection worker increments the buckets. + * In-flight / intermediate records never enter the projection. * * 1. HTTP Status Check: 2xx/3xx = success (green), other finalized HTTP status codes = failure (red) * @@ -17,6 +18,7 @@ export { AvailabilityQueryValidationError, calculateAvailabilityScore, classifyRequestStatus, + CURRENT_PROVIDER_STATUS_WINDOW_MINUTES, determineOptimalBucketSize, getCurrentProviderStatus, MAX_AVAILABILITY_QUERY_RANGE_DAYS, diff --git a/src/lib/availability/projection-tables.ts b/src/lib/availability/projection-tables.ts new file mode 100644 index 000000000..f945bb7d1 --- /dev/null +++ b/src/lib/availability/projection-tables.ts @@ -0,0 +1,85 @@ +/** + * Availability projection table definitions (outbox + 1m buckets). + * Kept in sync with drizzle/0120_availability_projection.sql and re-exported from schema.ts + * so drizzle-kit generate sees the same shape. + */ +import { sql } from "drizzle-orm"; +import { + bigint, + bigserial, + doublePrecision, + index, + integer, + jsonb, + pgTable, + primaryKey, + text, + timestamp, + unique, + uuid, +} from "drizzle-orm/pg-core"; + +export const outboxEvents = pgTable( + "outbox_events", + { + id: bigserial("id", { mode: "number" }).primaryKey(), + eventId: uuid("event_id").notNull().defaultRandom(), + eventType: text("event_type").notNull(), + aggregateType: text("aggregate_type").notNull(), + aggregateId: bigint("aggregate_id", { mode: "number" }).notNull(), + occurredAt: timestamp("occurred_at", { withTimezone: true }).notNull(), + payload: jsonb("payload").notNull(), + createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(), + publishedAt: timestamp("published_at", { withTimezone: true }), + attempts: integer("attempts").notNull().default(0), + lastError: text("last_error"), + }, + (t) => [ + unique("outbox_events_event_id_key").on(t.eventId), + index("idx_outbox_events_unpublished").on(t.id.asc()).where(sql`${t.publishedAt} IS NULL`), + ] +); + +export const outboxProcessed = pgTable("outbox_processed", { + eventId: uuid("event_id").primaryKey(), + processedAt: timestamp("processed_at", { withTimezone: true }).notNull().defaultNow(), +}); + +export const projAppliedRequests = pgTable("proj_applied_requests", { + requestId: bigint("request_id", { mode: "number" }).primaryKey(), + eventId: uuid("event_id").notNull(), + appliedAt: timestamp("applied_at", { withTimezone: true }).notNull().defaultNow(), +}); + +export const availBucket1m = pgTable( + "avail_bucket_1m", + { + providerId: integer("provider_id").notNull(), + bucketStart: timestamp("bucket_start", { withTimezone: true }).notNull(), + successCnt: integer("success_cnt").notNull().default(0), + failureCnt: integer("failure_cnt").notNull().default(0), + excludedCnt: integer("excluded_cnt").notNull().default(0), + latencyCnt: integer("latency_cnt").notNull().default(0), + latencySumMs: bigint("latency_sum_ms", { mode: "number" }).notNull().default(0), + lastRequestAt: timestamp("last_request_at", { withTimezone: true }), + }, + (t) => [ + primaryKey({ columns: [t.providerId, t.bucketStart], name: "avail_bucket_1m_pkey" }), + index("idx_avail_bucket_1m_time").on(t.bucketStart.desc()), + ] +); + +export const availCurrent = pgTable("avail_current", { + providerId: integer("provider_id").primaryKey(), + state: text("state").notNull().default("unknown"), + availability: doublePrecision("availability").notNull().default(0), + requestCount: integer("request_count").notNull().default(0), + lastRequestAt: timestamp("last_request_at", { withTimezone: true }), + updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), +}); + +export const projectionMeta = pgTable("projection_meta", { + key: text("key").primaryKey(), + value: jsonb("value").notNull().default({}), + updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), +}); diff --git a/src/lib/availability/projection-worker.ts b/src/lib/availability/projection-worker.ts new file mode 100644 index 000000000..a3e4d1cc7 --- /dev/null +++ b/src/lib/availability/projection-worker.ts @@ -0,0 +1,540 @@ +/** + * In-process outbox consumer for availability projection buckets. + * DB trigger on message_request writes outbox_events; this loop increments avail_bucket_1m. + */ +import "server-only"; + +import { sql } from "drizzle-orm"; +import { db } from "@/drizzle/db"; +import { CURRENT_PROVIDER_STATUS_WINDOW_MINUTES } from "@/lib/availability/availability-service"; +import { logger } from "@/lib/logger"; +import { withAdvisoryLock } from "@/lib/migrate"; + +const BATCH = 300; +const BUSY_MS = 10; +const TICK_MS = 200; +const BACKFILL_LOCK = "claude-code-hub:availability-projection-backfill"; +/** Match MAX_AVAILABILITY_QUERY_RANGE_DAYS so historical ranges are not silently empty after upgrade. */ +const BACKFILL_RANGE_DAYS = 100; +const BACKFILL_CHUNK_HOURS = 6; + +type SchedulerState = { + started?: boolean; + stopRequested?: boolean; + intervalId?: ReturnType; + currentPromise?: Promise; + bootstrapPromise?: Promise; +}; + +const schedulerState = globalThis as typeof globalThis & { + __CCH_AVAIL_PROJ_WORKER__?: SchedulerState; +}; + +function state(): SchedulerState { + if (!schedulerState.__CCH_AVAIL_PROJ_WORKER__) { + schedulerState.__CCH_AVAIL_PROJ_WORKER__ = {}; + } + return schedulerState.__CCH_AVAIL_PROJ_WORKER__; +} + +type ClaimedEvent = { + id: number; + event_id: string; + payload: { + request_id?: number | string; + provider_id?: number | string; + outcome?: string; + occurred_at?: string; + duration_ms?: number | string | null; + }; +}; + +export function asPayload(raw: unknown): ClaimedEvent["payload"] { + if (!raw) return {}; + if (typeof raw === "string") { + try { + return JSON.parse(raw) as ClaimedEvent["payload"]; + } catch { + return {}; + } + } + if (typeof raw === "object") { + return raw as ClaimedEvent["payload"]; + } + return {}; +} + +async function enqueueBackfillChunk(fromIso: string, toIso: string): Promise { + const result = await db.execute(sql` + WITH inserted AS ( + INSERT INTO outbox_events (event_type, aggregate_type, aggregate_id, occurred_at, payload) + SELECT + 'request_finalized', + 'message_request', + mr.id, + mr.created_at, + jsonb_build_object( + 'request_id', mr.id, + 'provider_id', mr.provider_id, + 'model', mr.model, + 'occurred_at', mr.created_at, + 'status_code', mr.status_code, + 'duration_ms', mr.duration_ms, + 'ttfb_ms', mr.ttfb_ms, + 'blocked_by', mr.blocked_by, + 'outcome', fn_compute_message_request_success_rate_outcome( + mr.blocked_by, mr.status_code, mr.error_message, mr.provider_chain + ), + 'group_tag', p.group_tag, + 'is_replay', COALESCE(mr.is_replay, false) + ) + FROM message_request mr + LEFT JOIN providers p ON p.id = mr.provider_id + WHERE mr.status_code IS NOT NULL + AND mr.created_at >= ${fromIso}::timestamptz + AND mr.created_at < ${toIso}::timestamptz + AND COALESCE(mr.is_replay, false) = false + AND NOT EXISTS (SELECT 1 FROM proj_applied_requests a WHERE a.request_id = mr.id) + AND fn_compute_message_request_success_rate_outcome( + mr.blocked_by, mr.status_code, mr.error_message, mr.provider_chain + ) IS NOT NULL + RETURNING 1 + ) + SELECT count(*)::int AS n FROM inserted + `); + const row = Array.from(result as Iterable<{ n?: number }>)[0]; + return Number(row?.n ?? 0); +} + +async function bootstrapBackfill(): Promise { + const existing = await db.execute(sql` + SELECT key FROM projection_meta WHERE key = 'backfill_done' LIMIT 1 + `); + if (Array.from(existing as Iterable).length > 0) { + return; + } + + const lockResult = await withAdvisoryLock( + BACKFILL_LOCK, + async () => { + // Re-check under lock so concurrent instances do not double-enqueue. + const again = await db.execute(sql` + SELECT key FROM projection_meta WHERE key = 'backfill_done' LIMIT 1 + `); + if (Array.from(again as Iterable).length > 0) { + return { skipped: true as const, inserted: 0 }; + } + + logger.info("[AvailProjection] starting backfill into outbox", { + rangeDays: BACKFILL_RANGE_DAYS, + chunkHours: BACKFILL_CHUNK_HOURS, + }); + + const endMs = Date.now(); + const startMs = endMs - BACKFILL_RANGE_DAYS * 24 * 60 * 60 * 1000; + const chunkMs = BACKFILL_CHUNK_HOURS * 60 * 60 * 1000; + let inserted = 0; + + for (let cursor = startMs; cursor < endMs; cursor += chunkMs) { + if (state().stopRequested) { + logger.warn("[AvailProjection] backfill interrupted by stop"); + break; + } + const fromIso = new Date(cursor).toISOString(); + const toIso = new Date(Math.min(cursor + chunkMs, endMs)).toISOString(); + inserted += await enqueueBackfillChunk(fromIso, toIso); + } + + if (!state().stopRequested) { + await db.execute(sql` + INSERT INTO projection_meta (key, value, updated_at) + VALUES ( + 'backfill_done', + jsonb_build_object( + 'at', now(), + 'note', 'availability backfill', + 'rangeDays', ${BACKFILL_RANGE_DAYS}, + 'inserted', ${inserted} + ), + now() + ) + ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now() + `); + logger.info("[AvailProjection] backfill enqueue finished", { inserted }); + } + + return { skipped: false as const, inserted }; + }, + { skipIfLocked: true } + ); + + if (!lockResult.ran) { + logger.info("[AvailProjection] backfill skipped; another instance holds the lock"); + } +} + +async function recomputeAvailCurrent(tx: typeof db, providerIds: number[]): Promise { + if (providerIds.length === 0) return; + + // Stable ascending lock order across concurrent worker instances (avoids deadlocks). + const sortedProviderIds = [...new Set(providerIds)].sort((a, b) => a - b); + const providerIdList = sql.join( + sortedProviderIds.map((id) => sql`${id}`), + sql`, ` + ); + + await tx.execute(sql` + INSERT INTO avail_current AS c ( + provider_id, state, availability, request_count, last_request_at, updated_at + ) + SELECT + s.provider_id, + s.state, + s.availability, + s.request_count, + s.last_request_at, + s.updated_at + FROM ( + SELECT + b.provider_id, + CASE + WHEN COALESCE(SUM(b.success_cnt), 0) + COALESCE(SUM(b.failure_cnt), 0) <= 0 THEN 'unknown' + WHEN (COALESCE(SUM(b.success_cnt), 0)::float + / (COALESCE(SUM(b.success_cnt), 0) + COALESCE(SUM(b.failure_cnt), 0))) >= 0.8 THEN 'green' + WHEN (COALESCE(SUM(b.success_cnt), 0)::float + / (COALESCE(SUM(b.success_cnt), 0) + COALESCE(SUM(b.failure_cnt), 0))) >= 0.5 THEN 'yellow' + ELSE 'red' + END AS state, + CASE + WHEN COALESCE(SUM(b.success_cnt), 0) + COALESCE(SUM(b.failure_cnt), 0) <= 0 THEN 0 + ELSE COALESCE(SUM(b.success_cnt), 0)::float + / (COALESCE(SUM(b.success_cnt), 0) + COALESCE(SUM(b.failure_cnt), 0)) + END AS availability, + (COALESCE(SUM(b.success_cnt), 0) + COALESCE(SUM(b.failure_cnt), 0))::int AS request_count, + MAX(b.last_request_at) AS last_request_at, + now() AS updated_at + FROM avail_bucket_1m b + WHERE b.provider_id IN (${providerIdList}) + AND b.bucket_start >= now() - (${sql.raw(String(CURRENT_PROVIDER_STATUS_WINDOW_MINUTES))} * INTERVAL '1 minute') + GROUP BY b.provider_id + ORDER BY b.provider_id ASC + ) s + ON CONFLICT (provider_id) DO UPDATE SET + state = EXCLUDED.state, + availability = EXCLUDED.availability, + request_count = EXCLUDED.request_count, + last_request_at = EXCLUDED.last_request_at, + updated_at = now() + `); + + // Providers with no traffic in the window become unknown (honest empty state). + // Lock target rows in provider_id order before updating. + await tx.execute(sql` + WITH targets AS ( + SELECT c.provider_id + FROM avail_current c + WHERE c.provider_id IN (${providerIdList}) + AND NOT EXISTS ( + SELECT 1 + FROM avail_bucket_1m b + WHERE b.provider_id = c.provider_id + AND b.bucket_start >= now() - (${sql.raw(String(CURRENT_PROVIDER_STATUS_WINDOW_MINUTES))} * INTERVAL '1 minute') + AND (b.success_cnt + b.failure_cnt) > 0 + ) + ORDER BY c.provider_id ASC + FOR UPDATE OF c + ) + UPDATE avail_current c + SET + state = 'unknown', + availability = 0, + request_count = 0, + updated_at = now() + FROM targets t + WHERE c.provider_id = t.provider_id + `); +} + +export async function processBatch(): Promise { + return await db.transaction(async (tx) => { + const claimedRows = await tx.execute(sql` + SELECT id, event_id, payload + FROM outbox_events + WHERE published_at IS NULL + ORDER BY id + FOR UPDATE SKIP LOCKED + LIMIT ${BATCH} + `); + + const claimed = Array.from(claimedRows as Iterable); + if (claimed.length === 0) { + return 0; + } + + let applied = 0; + const touchedProviders = new Set(); + const publishedIds: number[] = []; + const invalidIds: number[] = []; + + // Bucket deltas aggregated in JS, then one upsert per distinct (provider, minute). + type BucketKey = string; + const bucketDeltas = new Map< + BucketKey, + { + providerId: number; + bucketStartIso: string; + successCnt: number; + failureCnt: number; + excludedCnt: number; + latencyCnt: number; + latencySum: number; + lastRequestAtIso: string; + } + >(); + + for (const row of claimed) { + const payload = asPayload(row.payload); + const requestId = Number(payload.request_id); + const providerId = Number(payload.provider_id); + const outcome = String(payload.outcome || "excluded"); + const occurredAt = payload.occurred_at; + if (!Number.isFinite(requestId) || !Number.isFinite(providerId) || !occurredAt) { + invalidIds.push(row.id); + continue; + } + + const inserted = await tx.execute(sql` + INSERT INTO proj_applied_requests (request_id, event_id) + VALUES (${requestId}, ${row.event_id}::uuid) + ON CONFLICT (request_id) DO NOTHING + RETURNING request_id + `); + const isFresh = Array.from(inserted as Iterable).length > 0; + + if (isFresh) { + const durationMs = + payload.duration_ms === null || payload.duration_ms === undefined + ? null + : Number(payload.duration_ms); + const successCnt = outcome === "success" ? 1 : 0; + const failureCnt = outcome === "failure" ? 1 : 0; + const excludedCnt = outcome === "excluded" ? 1 : 0; + const latencyCnt = + (outcome === "success" || outcome === "failure") && + durationMs !== null && + Number.isFinite(durationMs) + ? 1 + : 0; + const latencySum = + latencyCnt === 1 && durationMs !== null && Number.isFinite(durationMs) + ? Math.trunc(durationMs) + : 0; + + const occurred = new Date(occurredAt); + const bucketStart = new Date( + Date.UTC( + occurred.getUTCFullYear(), + occurred.getUTCMonth(), + occurred.getUTCDate(), + occurred.getUTCHours(), + occurred.getUTCMinutes(), + 0, + 0 + ) + ); + const bucketStartIso = bucketStart.toISOString(); + const key = `${providerId}|${bucketStartIso}`; + const prev = bucketDeltas.get(key); + if (prev) { + prev.successCnt += successCnt; + prev.failureCnt += failureCnt; + prev.excludedCnt += excludedCnt; + prev.latencyCnt += latencyCnt; + prev.latencySum += latencySum; + if (occurredAt > prev.lastRequestAtIso) { + prev.lastRequestAtIso = occurredAt; + } + } else { + bucketDeltas.set(key, { + providerId, + bucketStartIso, + successCnt, + failureCnt, + excludedCnt, + latencyCnt, + latencySum, + lastRequestAtIso: occurredAt, + }); + } + touchedProviders.add(providerId); + applied += 1; + } + + publishedIds.push(row.id); + } + + // Upsert buckets in (provider_id, bucket_start) order so concurrent workers take locks consistently. + const sortedDeltas = Array.from(bucketDeltas.values()).sort((a, b) => { + if (a.providerId !== b.providerId) return a.providerId - b.providerId; + return a.bucketStartIso < b.bucketStartIso ? -1 : a.bucketStartIso > b.bucketStartIso ? 1 : 0; + }); + for (const delta of sortedDeltas) { + await tx.execute(sql` + INSERT INTO avail_bucket_1m AS b ( + provider_id, + bucket_start, + success_cnt, + failure_cnt, + excluded_cnt, + latency_cnt, + latency_sum_ms, + last_request_at + ) VALUES ( + ${delta.providerId}, + ${delta.bucketStartIso}::timestamptz, + ${delta.successCnt}, + ${delta.failureCnt}, + ${delta.excludedCnt}, + ${delta.latencyCnt}, + ${delta.latencySum}, + ${delta.lastRequestAtIso}::timestamptz + ) + ON CONFLICT (provider_id, bucket_start) DO UPDATE SET + success_cnt = b.success_cnt + EXCLUDED.success_cnt, + failure_cnt = b.failure_cnt + EXCLUDED.failure_cnt, + excluded_cnt = b.excluded_cnt + EXCLUDED.excluded_cnt, + latency_cnt = b.latency_cnt + EXCLUDED.latency_cnt, + latency_sum_ms = b.latency_sum_ms + EXCLUDED.latency_sum_ms, + last_request_at = GREATEST( + COALESCE(b.last_request_at, EXCLUDED.last_request_at), + EXCLUDED.last_request_at + ) + `); + } + + if (publishedIds.length > 0) { + const sortedPublishedIds = [...publishedIds].sort((a, b) => a - b); + await tx.execute(sql` + UPDATE outbox_events + SET published_at = now(), + attempts = attempts + 1, + last_error = NULL + WHERE id IN (${sql.join( + sortedPublishedIds.map((id) => sql`${id}`), + sql`, ` + )}) + `); + } + + if (invalidIds.length > 0) { + const sortedInvalidIds = [...invalidIds].sort((a, b) => a - b); + await tx.execute(sql` + UPDATE outbox_events + SET published_at = now(), + attempts = attempts + 1, + last_error = 'invalid payload' + WHERE id IN (${sql.join( + sortedInvalidIds.map((id) => sql`${id}`), + sql`, ` + )}) + `); + } + + await recomputeAvailCurrent(tx as unknown as typeof db, Array.from(touchedProviders)); + + return applied; + }); +} + +async function runCycle(): Promise { + const s = state(); + if (s.stopRequested) return; + if (s.currentPromise) return; + + let current!: Promise; + current = (async () => { + try { + let total = 0; + for (let i = 0; i < 20; i++) { + if (s.stopRequested) break; + const n = await processBatch(); + total += n; + if (n === 0) break; + if (n < BATCH) break; + await new Promise((r) => setTimeout(r, BUSY_MS)); + } + if (total > 0) { + logger.info("[AvailProjection] projected events", { count: total }); + } + } catch (error) { + logger.warn("[AvailProjection] cycle failed", { + error: error instanceof Error ? error.message : String(error), + }); + } finally { + if (s.currentPromise === current) { + s.currentPromise = undefined; + } + } + })(); + + s.currentPromise = current; + await current; +} + +export function startAvailabilityProjectionWorker(): void { + const s = state(); + if (s.started) return; + + s.stopRequested = false; + s.started = true; + + s.bootstrapPromise = (async () => { + try { + await bootstrapBackfill(); + } catch (error) { + logger.warn("[AvailProjection] backfill failed", { + error: error instanceof Error ? error.message : String(error), + }); + } + void runCycle(); + })(); + + s.intervalId = setInterval(() => { + void runCycle(); + }, TICK_MS); + (s.intervalId as { unref?: () => void } | undefined)?.unref?.(); + + logger.info("[AvailProjection] worker started"); +} + +export async function stopAvailabilityProjectionWorker(): Promise { + const s = state(); + s.stopRequested = true; + if (s.intervalId) { + clearInterval(s.intervalId); + s.intervalId = undefined; + } + await s.bootstrapPromise; + await s.currentPromise; + s.started = false; + s.bootstrapPromise = undefined; + logger.info("[AvailProjection] worker stopped"); +} + +export function getAvailabilityProjectionWorkerStatus() { + const s = state(); + return { + started: s.started === true, + running: Boolean(s.currentPromise), + bootstrapping: Boolean(s.bootstrapPromise), + tickMs: TICK_MS, + }; +} + +/** Test-only helpers */ +export const __test__ = { + bootstrapBackfill, + recomputeAvailCurrent, + BACKFILL_RANGE_DAYS, + BATCH, +}; diff --git a/src/lib/availability/types.ts b/src/lib/availability/types.ts index 281aa9e36..e78ca466b 100644 --- a/src/lib/availability/types.ts +++ b/src/lib/availability/types.ts @@ -63,11 +63,15 @@ export interface TimeBucketMetrics { availabilityScore: number; /** Average latency in ms */ avgLatencyMs: number; - /** P50 latency in ms */ + /** + * Latency percentile fields kept for API compatibility. + * With 1m sum/count projection buckets these currently equal avgLatencyMs + * (mean approximation) until sketch/histogram-based percentiles land. + */ p50LatencyMs: number; - /** P95 latency in ms */ + /** @see p50LatencyMs */ p95LatencyMs: number; - /** P99 latency in ms */ + /** @see p50LatencyMs */ p99LatencyMs: number; } diff --git a/src/lib/lifecycle/shutdown.ts b/src/lib/lifecycle/shutdown.ts index 23443ee95..79e053c4c 100644 --- a/src/lib/lifecycle/shutdown.ts +++ b/src/lib/lifecycle/shutdown.ts @@ -191,7 +191,26 @@ export async function runApplicationCleanup( clearTimeout(asyncTasksWarningTimer); } - // 7. 刷写 message_request 异步写缓冲。这里不能用可脱离的单步 timeout： + // 7a. 可用性投影 worker 在 closeDbPools 前必须真正停住（含 backfill / 在飞 batch）。 + // 超时只告警，不能 detach；否则会在投影事务进行中关掉 DB。 + const availProjWarningTimer = setTimeout(() => { + logger.warn("[Shutdown] stopAvailabilityProjectionWorker still pending", { ms: stepMs }); + }, stepMs); + try { + const { stopAvailabilityProjectionWorker } = await import( + "@/lib/availability/projection-worker" + ); + await stopAvailabilityProjectionWorker(); + } catch (error) { + logger.error("[Shutdown] availability projection worker failed to stop", { + error: error instanceof Error ? error.message : String(error), + }); + throw error; + } finally { + clearTimeout(availProjWarningTimer); + } + + // 7b. 刷写 message_request 异步写缓冲。这里不能用可脱离的单步 timeout： // closeDbPools 必须等 writer 真正 settled，否则会关闭仍在执行终态 SQL 的连接。 writerQuiescencePending = true; const writerWarningTimer = setTimeout(() => { diff --git a/tests/unit/lib/availability-service.test.ts b/tests/unit/lib/availability-service.test.ts index fcae47fde..8c25249ad 100644 --- a/tests/unit/lib/availability-service.test.ts +++ b/tests/unit/lib/availability-service.test.ts @@ -39,26 +39,14 @@ function normalizeSql(sqlObject: unknown): string { return sqlToString(sqlObject).replace(/\s+/g, " ").trim().toLowerCase(); } -function extractFinalizedRequestsSql(queryText: string): string { - const start = queryText.indexOf("finalized_requests as"); - const end = queryText.indexOf("provider_bucket_stats as"); - - if (start === -1 || end === -1 || end <= start) { - throw new Error("Could not locate finalized_requests CTE in query text"); - } - - return queryText.slice(start, end); -} - -// 终态边界必须仅由 status_code 收敛：不能回退到包含 blocked_by / -// error_message / provider_chain 任一非空的旧语义，否则会重新把"请求中" -// 记录纳入可用性统计。每一处断言都重复这套规则，防止个别用例漏检导致回归。 -function expectStatusCodeOnlyFinalizedBoundary(sqlText: string) { +function expectProjectionBucketReadPath(sqlText: string) { + expect(sqlText).toContain("from avail_bucket_1m"); + expect(sqlText).toContain("date_bin"); + expect(sqlText).toContain("row_number() over"); + expect(sqlText).not.toContain("from message_request"); + expect(sqlText).not.toContain("fn_compute_message_request_success_rate_outcome"); + expect(sqlText).not.toContain("percentile_cont"); expect(sqlText).not.toContain("fn_is_message_request_finalized"); - expect(sqlText).toContain(`"status_code" is not null`); - expect(sqlText).not.toContain(`"blocked_by" is not null`); - expect(sqlText).not.toContain(`"error_message" is not null`); - expect(sqlText).not.toContain(`"provider_chain" -> -1 ->> 'reason'`); } describe("availability-service", () => { @@ -253,7 +241,41 @@ describe("availability-service", () => { expect(executeMock).not.toHaveBeenCalled(); }); - it("queryProviderAvailability 改为数据库聚合后仍只统计终态请求", async () => { + it("queryProviderAvailability 用 floor 到分钟的边界包含部分分钟桶", async () => { + const selectMock = vi.fn(() => + createThenableQuery([ + { + id: 1, + name: "Provider A", + providerType: "claude", + enabled: true, + }, + ]) + ); + const executeMock = vi.fn(async () => []); + + vi.doMock("@/drizzle/db", () => ({ + db: { + select: selectMock, + execute: executeMock, + }, + })); + + const { queryProviderAvailability } = await import("@/lib/availability/availability-service"); + await queryProviderAvailability({ + startTime: new Date("2026-04-13T07:00:30.000Z"), + endTime: new Date("2026-04-13T09:00:45.000Z"), + bucketSizeMinutes: 60, + }); + + const query = sqlToQuery(executeMock.mock.calls[0]?.[0]); + // floor start -> 07:00:00, floor end -> 09:00:00 + expect(query.params).toContain("2026-04-13T07:00:00.000Z"); + expect(query.params).toContain("2026-04-13T09:00:00.000Z"); + expect(query.params).not.toContain("2026-04-13T07:00:30.000Z"); + }); + + it("queryProviderAvailability 从 avail_bucket_1m 投影表聚合，不再扫描 message_request", async () => { const selectMock = vi.fn(() => createThenableQuery([ { @@ -319,21 +341,9 @@ describe("availability-service", () => { }); const queryText = normalizeSql(executeMock.mock.calls[0]?.[0]); - const finalizedRequestsSql = extractFinalizedRequestsSql(queryText); - // 可用性监控的终态边界收敛为 status_code IS NOT NULL， - // 这样才能命中部分索引 idx_message_request_provider_created_at_finalized_active； - // 同时不会把 providerChain / errorMessage 已写入但 statusCode 仍为空的"请求中" - // 记录纳入聚合 —— 它们会在分类阶段被误判成 failure。 - expectStatusCodeOnlyFinalizedBoundary(finalizedRequestsSql); - expect(queryText).toContain("group by"); - expect(queryText).toContain("percentile_cont(0.95)"); - expect(queryText).toContain("row_number() over"); - expect(queryText).toContain(`"successrateoutcome" in ('success', 'failure')`); - // 终态记录的 success/failure/excluded 分类仍由 outcome 函数完成。 - expect(queryText).toContain("fn_compute_message_request_success_rate_outcome"); - expect(queryText).toContain('avg("durationms") filter'); - expect(queryText).toContain('"message_request"."is_replay" ='); - expect(sqlToQuery(executeMock.mock.calls[0]?.[0]).params).toContain(false); + expectProjectionBucketReadPath(queryText); + expect(queryText).toContain("where rn <="); + expect(sqlToQuery(executeMock.mock.calls[0]?.[0]).params).toContain(60); }); it("queryProviderAvailability 计算 currentStatus 时会按最近 buckets 的请求量加权", async () => { @@ -431,8 +441,9 @@ describe("availability-service", () => { expect(selectMock).toHaveBeenCalledTimes(1); expect(executeMock).toHaveBeenCalledTimes(1); expect(result.bucketSizeMinutes).toBe(5); - expect(query.params).toContain(300); + expect(query.params).toContain(5); expect(query.params).not.toContain(Number.POSITIVE_INFINITY); + expectProjectionBucketReadPath(normalizeSql(executeMock.mock.calls[0]?.[0])); }); it("queryProviderAvailability 在 bucketSizeMinutes 为超大有限值时钳制到 1440 分钟", async () => { @@ -467,117 +478,8 @@ describe("availability-service", () => { expect(selectMock).toHaveBeenCalledTimes(1); expect(executeMock).toHaveBeenCalledTimes(1); expect(result.bucketSizeMinutes).toBe(1440); - expect(query.params).toContain(86400); - expect(query.params).not.toContain(Number.MAX_SAFE_INTEGER * 60); - }); - - it("queryProviderAvailability 会排除进行中请求(statusCode=null 且 durationMs=null)", async () => { - const selectMock = vi.fn(() => - createThenableQuery([ - { - id: 1, - name: "Provider A", - providerType: "claude", - enabled: true, - }, - ]) - ); - const executeMock = vi.fn(async () => []); - - vi.doMock("@/drizzle/db", () => ({ - db: { - select: selectMock, - execute: executeMock, - }, - })); - - const { queryProviderAvailability } = await import("@/lib/availability/availability-service"); - await queryProviderAvailability({ - startTime: new Date("2026-04-13T07:00:00.000Z"), - endTime: new Date("2026-04-13T09:00:00.000Z"), - bucketSizeMinutes: 60, - }); - - const finalizedRequestsSql = extractFinalizedRequestsSql( - normalizeSql(executeMock.mock.calls[0]?.[0]) - ); - // 终态判定只看 status_code IS NOT NULL：要么命中部分索引，要么直接排除"请求中" - // 的记录，不再依据 providerChain / errorMessage 片段把它们判为终态。 - expectStatusCodeOnlyFinalizedBoundary(finalizedRequestsSql); - }); - - it("queryProviderAvailability 会保留 Gemini passthrough 终态(statusCode!=null 且 durationMs=null)", async () => { - const selectMock = vi.fn(() => - createThenableQuery([ - { - id: 1, - name: "Provider A", - providerType: "claude", - enabled: true, - }, - ]) - ); - const executeMock = vi.fn(async () => []); - - vi.doMock("@/drizzle/db", () => ({ - db: { - select: selectMock, - execute: executeMock, - }, - })); - - const { queryProviderAvailability } = await import("@/lib/availability/availability-service"); - await queryProviderAvailability({ - startTime: new Date("2026-04-13T07:00:00.000Z"), - endTime: new Date("2026-04-13T09:00:00.000Z"), - bucketSizeMinutes: 60, - }); - - const finalizedRequestsSql = extractFinalizedRequestsSql( - normalizeSql(executeMock.mock.calls[0]?.[0]) - ); - expect(finalizedRequestsSql).not.toMatch(/where .*duration_?ms.*is not null/); - // Gemini passthrough 写入了 statusCode（即使 durationMs 仍为 null）， - // 因此会被 status_code IS NOT NULL 的终态过滤保留下来；同时保持终态边界 - // 不被其他字段放宽。 - expectStatusCodeOnlyFinalizedBoundary(finalizedRequestsSql); - }); - - it("queryProviderAvailability 当前不会把中间持久化状态(statusCode=null 且 durationMs!=null)误算为 red", async () => { - const selectMock = vi.fn(() => - createThenableQuery([ - { - id: 1, - name: "Provider A", - providerType: "claude", - enabled: true, - }, - ]) - ); - const executeMock = vi.fn(async () => []); - - vi.doMock("@/drizzle/db", () => ({ - db: { - select: selectMock, - execute: executeMock, - }, - })); - - const { queryProviderAvailability } = await import("@/lib/availability/availability-service"); - await queryProviderAvailability({ - startTime: new Date("2026-04-13T07:00:00.000Z"), - endTime: new Date("2026-04-13T09:00:00.000Z"), - bucketSizeMinutes: 60, - }); - - const queryText = normalizeSql(executeMock.mock.calls[0]?.[0]); - const finalizedRequestsSql = extractFinalizedRequestsSql(queryText); - - // status_code IS NOT NULL 把 statusCode=null 的中间持久化记录直接排除在聚合外， - // 它们根本不会进入 outcome 分类阶段，所以不会被算成 failure。 - expectStatusCodeOnlyFinalizedBoundary(finalizedRequestsSql); - expect(queryText).toContain("fn_compute_message_request_success_rate_outcome"); - expect(queryText).toContain(`"successrateoutcome" = 'failure'`); + expect(query.params).toContain(1440); + expect(query.params).not.toContain(Number.MAX_SAFE_INTEGER); }); it("queryProviderAvailability 在 maxBuckets 为 Infinity 时仍使用默认桶上限", async () => { @@ -702,7 +604,7 @@ describe("availability-service", () => { ]); }); - it("getCurrentProviderStatus 改为数据库聚合后仍只统计终态请求", async () => { + it("getCurrentProviderStatus 优先读取 avail_current 投影表", async () => { const selectMock = vi.fn(() => createThenableQuery([ { @@ -711,12 +613,15 @@ describe("availability-service", () => { }, ]) ); + const fresh = new Date(); const executeMock = vi.fn(async () => [ { providerId: 1, - greenCount: 1, - redCount: 1, - lastRequestAt: new Date("2026-04-13T08:02:00.000Z"), + state: "green", + availability: 0.5, + requestCount: 2, + lastRequestAt: fresh, + updatedAt: fresh, }, ]); @@ -739,21 +644,112 @@ describe("availability-service", () => { status: "green", availability: 0.5, requestCount: 2, - lastRequestAt: "2026-04-13T08:02:00.000Z", + lastRequestAt: fresh.toISOString(), }, ]); const queryText = normalizeSql(executeMock.mock.calls[0]?.[0]); - // getCurrentProviderStatus 同样使用 status_code IS NOT NULL 终态边界， - // 让短窗口查询也能直接命中部分索引并避免误判"请求中"。 - expectStatusCodeOnlyFinalizedBoundary(queryText); - expect(queryText).toContain("fn_compute_message_request_success_rate_outcome"); - expect(queryText).toContain(">= now() - (15 * interval '1 minute')"); - expect(queryText).toContain("<= now()"); - expect(queryText).toContain("count(*) filter"); - expect(queryText).toContain("max("); - expect(queryText).toContain('"message_request"."is_replay" ='); - expect(sqlToQuery(executeMock.mock.calls[0]?.[0]).params).toContain(false); + expect(queryText).toContain("from avail_current"); + expect(queryText).toContain("updated_at"); + expect(queryText).not.toContain("from message_request"); + expect(queryText).not.toContain("fn_compute_message_request_success_rate_outcome"); + }); + + it("getCurrentProviderStatus 对过期 avail_current 行返回 unknown（或走桶回退）", async () => { + const selectMock = vi.fn(() => + createThenableQuery([ + { + id: 1, + name: "Provider A", + }, + ]) + ); + const stale = new Date(Date.now() - 60 * 60 * 1000); + const executeMock = vi + .fn() + .mockResolvedValueOnce([ + { + providerId: 1, + state: "green", + availability: 1, + requestCount: 9, + lastRequestAt: stale, + updatedAt: stale, + }, + ]) + .mockResolvedValueOnce([]); + + vi.doMock("@/drizzle/db", () => ({ + db: { + select: selectMock, + execute: executeMock, + }, + })); + + const { getCurrentProviderStatus } = await import("@/lib/availability/availability-service"); + const result = await getCurrentProviderStatus(); + + expect(executeMock).toHaveBeenCalledTimes(2); + expect(result).toEqual([ + { + providerId: 1, + providerName: "Provider A", + status: "unknown", + availability: 0, + requestCount: 0, + lastRequestAt: null, + }, + ]); + }); + + it("getCurrentProviderStatus 在 avail_current 缺失时回退到 avail_bucket_1m", async () => { + const selectMock = vi.fn(() => + createThenableQuery([ + { + id: 1, + name: "Provider A", + }, + ]) + ); + const executeMock = vi + .fn() + .mockResolvedValueOnce([]) + .mockResolvedValueOnce([ + { + providerId: 1, + greenCount: 3, + redCount: 1, + lastRequestAt: new Date("2026-04-13T08:05:00.000Z"), + }, + ]); + + vi.doMock("@/drizzle/db", () => ({ + db: { + select: selectMock, + execute: executeMock, + }, + })); + + const { getCurrentProviderStatus } = await import("@/lib/availability/availability-service"); + const result = await getCurrentProviderStatus(); + + expect(executeMock).toHaveBeenCalledTimes(2); + expect(result).toEqual([ + { + providerId: 1, + providerName: "Provider A", + status: "green", + availability: 0.75, + requestCount: 4, + lastRequestAt: "2026-04-13T08:05:00.000Z", + }, + ]); + + expect(normalizeSql(executeMock.mock.calls[0]?.[0])).toContain("from avail_current"); + expect(normalizeSql(executeMock.mock.calls[1]?.[0])).toContain("from avail_bucket_1m"); + expect(normalizeSql(executeMock.mock.calls[1]?.[0])).toContain( + ">= now() - (15 * interval '1 minute')" + ); }); it("getCurrentProviderStatus 在提供商无聚合数据时返回 unknown", async () => { @@ -765,7 +761,8 @@ describe("availability-service", () => { }, ]) ); - const executeMock = vi.fn(async () => []); + // first avail_current empty, then fallback empty + const executeMock = vi.fn().mockResolvedValueOnce([]).mockResolvedValueOnce([]); vi.doMock("@/drizzle/db", () => ({ db: { @@ -778,7 +775,7 @@ describe("availability-service", () => { const result = await getCurrentProviderStatus(); expect(selectMock).toHaveBeenCalledTimes(1); - expect(executeMock).toHaveBeenCalledTimes(1); + expect(executeMock).toHaveBeenCalledTimes(2); expect(result).toEqual([ { providerId: 1, diff --git a/tests/unit/lib/availability/projection-worker.test.ts b/tests/unit/lib/availability/projection-worker.test.ts new file mode 100644 index 000000000..baa653709 --- /dev/null +++ b/tests/unit/lib/availability/projection-worker.test.ts @@ -0,0 +1,210 @@ +import type { SQL } from "drizzle-orm"; +import { CasingCache } from "drizzle-orm/casing"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +function sqlToString(sqlObject: unknown): string { + return (sqlObject as SQL) + .toQuery({ + escapeName: (name: string) => `"${name}"`, + escapeParam: (num: number, _value: unknown) => `$${num}`, + escapeString: (value: string) => `'${value}'`, + casing: new CasingCache(), + paramStartIndex: { value: 1 }, + }) + .sql.replace(/\s+/g, " ") + .trim() + .toLowerCase(); +} + +describe("availability projection-worker", () => { + beforeEach(() => { + vi.resetModules(); + vi.clearAllMocks(); + delete (globalThis as { __CCH_AVAIL_PROJ_WORKER__?: unknown }).__CCH_AVAIL_PROJ_WORKER__; + }); + + it("asPayload 解析 object / JSON 字符串 / 非法输入", async () => { + vi.doMock("@/drizzle/db", () => ({ + db: { execute: vi.fn(), transaction: vi.fn() }, + })); + vi.doMock("@/lib/migrate", () => ({ + withAdvisoryLock: vi.fn(async (_n: string, fn: () => Promise) => ({ + ran: true, + result: await fn(), + })), + })); + vi.doMock("@/lib/logger", () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + })); + + const { asPayload } = await import("@/lib/availability/projection-worker"); + + expect(asPayload({ request_id: 1, provider_id: 2 })).toEqual({ + request_id: 1, + provider_id: 2, + }); + expect(asPayload('{"request_id":3}')).toEqual({ request_id: 3 }); + expect(asPayload("{not-json")).toEqual({}); + expect(asPayload(null)).toEqual({}); + expect(asPayload(42)).toEqual({}); + }); + + it("processBatch 对新鲜事件写入 1m 桶并重算 avail_current", async () => { + const executeMock = vi.fn(async (query: unknown) => { + const text = sqlToString(query); + if (text.includes("for update skip locked")) { + return [ + { + id: 10, + event_id: "11111111-1111-1111-1111-111111111111", + payload: { + request_id: 100, + provider_id: 7, + outcome: "success", + occurred_at: "2026-04-13T08:03:12.000Z", + duration_ms: 120, + }, + }, + ]; + } + if (text.includes("insert into proj_applied_requests")) { + return [{ request_id: 100 }]; + } + return []; + }); + const transactionMock = vi.fn(async (fn: (tx: { execute: typeof executeMock }) => Promise) => + fn({ execute: executeMock }) + ); + + vi.doMock("@/drizzle/db", () => ({ + db: { execute: executeMock, transaction: transactionMock }, + })); + vi.doMock("@/lib/migrate", () => ({ + withAdvisoryLock: vi.fn(async (_n: string, fn: () => Promise) => ({ + ran: true, + result: await fn(), + })), + })); + vi.doMock("@/lib/logger", () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + })); + + const { processBatch } = await import("@/lib/availability/projection-worker"); + const applied = await processBatch(); + expect(applied).toBe(1); + + const texts = executeMock.mock.calls.map((c) => sqlToString(c[0])); + expect(texts.some((t) => t.includes("insert into avail_bucket_1m"))).toBe(true); + expect(texts.some((t) => t.includes("insert into avail_current"))).toBe(true); + expect(texts.some((t) => t.includes("15 * interval '1 minute'"))).toBe(true); + expect(texts.some((t) => t.includes("update outbox_events") && t.includes("published_at"))).toBe( + true + ); + }); + + it("processBatch 对重复 request 不重复计数", async () => { + const executeMock = vi.fn(async (query: unknown) => { + const text = sqlToString(query); + if (text.includes("for update skip locked")) { + return [ + { + id: 11, + event_id: "22222222-2222-2222-2222-222222222222", + payload: { + request_id: 100, + provider_id: 7, + outcome: "success", + occurred_at: "2026-04-13T08:03:12.000Z", + duration_ms: 120, + }, + }, + ]; + } + if (text.includes("insert into proj_applied_requests")) { + return []; // ON CONFLICT DO NOTHING + } + return []; + }); + const transactionMock = vi.fn(async (fn: (tx: { execute: typeof executeMock }) => Promise) => + fn({ execute: executeMock }) + ); + + vi.doMock("@/drizzle/db", () => ({ + db: { execute: executeMock, transaction: transactionMock }, + })); + vi.doMock("@/lib/migrate", () => ({ + withAdvisoryLock: vi.fn(async (_n: string, fn: () => Promise) => ({ + ran: true, + result: await fn(), + })), + })); + vi.doMock("@/lib/logger", () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + })); + + const { processBatch } = await import("@/lib/availability/projection-worker"); + const applied = await processBatch(); + expect(applied).toBe(0); + + const texts = executeMock.mock.calls.map((c) => sqlToString(c[0])); + expect(texts.some((t) => t.includes("insert into avail_bucket_1m"))).toBe(false); + expect(texts.some((t) => t.includes("update outbox_events"))).toBe(true); + }); + + it("processBatch 将非法 payload 标记 published + last_error", async () => { + const executeMock = vi.fn(async (query: unknown) => { + const text = sqlToString(query); + if (text.includes("for update skip locked")) { + return [ + { + id: 12, + event_id: "33333333-3333-3333-3333-333333333333", + payload: { outcome: "success" }, + }, + ]; + } + return []; + }); + const transactionMock = vi.fn(async (fn: (tx: { execute: typeof executeMock }) => Promise) => + fn({ execute: executeMock }) + ); + + vi.doMock("@/drizzle/db", () => ({ + db: { execute: executeMock, transaction: transactionMock }, + })); + vi.doMock("@/lib/migrate", () => ({ + withAdvisoryLock: vi.fn(async (_n: string, fn: () => Promise) => ({ + ran: true, + result: await fn(), + })), + })); + vi.doMock("@/lib/logger", () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + })); + + const { processBatch } = await import("@/lib/availability/projection-worker"); + expect(await processBatch()).toBe(0); + + const texts = executeMock.mock.calls.map((c) => sqlToString(c[0])); + expect(texts.some((t) => t.includes("last_error") && t.includes("invalid payload"))).toBe(true); + }); + + it("bootstrapBackfill 在 backfill_done 已存在时为 no-op", async () => { + const executeMock = vi.fn(async () => [{ key: "backfill_done" }]); + const withAdvisoryLock = vi.fn(); + + vi.doMock("@/drizzle/db", () => ({ + db: { execute: executeMock, transaction: vi.fn() }, + })); + vi.doMock("@/lib/migrate", () => ({ withAdvisoryLock })); + vi.doMock("@/lib/logger", () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + })); + + const { __test__ } = await import("@/lib/availability/projection-worker"); + await __test__.bootstrapBackfill(); + + expect(executeMock).toHaveBeenCalledTimes(1); + expect(withAdvisoryLock).not.toHaveBeenCalled(); + }); +}); diff --git a/tests/unit/lib/shutdown.test.ts b/tests/unit/lib/shutdown.test.ts index 70b80e612..96b415327 100644 --- a/tests/unit/lib/shutdown.test.ts +++ b/tests/unit/lib/shutdown.test.ts @@ -66,6 +66,7 @@ describe.sequential("lifecycle/shutdown", () => { throw new Error("simulated probe scheduler shutdown failure"); }); const stopPublicStatus = vi.fn(async () => {}); + const stopAvailProj = vi.fn(async () => {}); const stopProbeLog = vi.fn(); const shutdownTasks = vi.fn(async () => {}); const stopWriteBuffer = vi.fn(async () => {}); @@ -79,6 +80,9 @@ describe.sequential("lifecycle/shutdown", () => { vi.doMock("@/lib/public-status/scheduler", () => ({ stopPublicStatusRebuildScheduler: stopPublicStatus, })); + vi.doMock("@/lib/availability/projection-worker", () => ({ + stopAvailabilityProjectionWorker: stopAvailProj, + })); vi.doMock("@/lib/provider-endpoints/probe-log-cleanup", () => ({ stopEndpointProbeLogCleanup: stopProbeLog, })); @@ -109,6 +113,7 @@ describe.sequential("lifecycle/shutdown", () => { expect(stopCache).toHaveBeenCalled(); expect(stopProbe).toHaveBeenCalled(); expect(stopPublicStatus).toHaveBeenCalled(); + expect(stopAvailProj).toHaveBeenCalled(); expect(stopProbeLog).toHaveBeenCalled(); expect(shutdownTasks).toHaveBeenCalled(); expect(stopWriteBuffer).toHaveBeenCalled(); @@ -133,6 +138,9 @@ describe.sequential("lifecycle/shutdown", () => { vi.doMock("@/lib/public-status/scheduler", () => ({ stopPublicStatusRebuildScheduler: async () => {}, })); + vi.doMock("@/lib/availability/projection-worker", () => ({ + stopAvailabilityProjectionWorker: async () => {}, + })); vi.doMock("@/lib/provider-endpoints/probe-log-cleanup", () => ({ stopEndpointProbeLogCleanup: async () => {}, })); @@ -174,6 +182,9 @@ describe.sequential("lifecycle/shutdown", () => { vi.doMock("@/lib/public-status/scheduler", () => ({ stopPublicStatusRebuildScheduler: async () => {}, })); + vi.doMock("@/lib/availability/projection-worker", () => ({ + stopAvailabilityProjectionWorker: async () => {}, + })); vi.doMock("@/lib/provider-endpoints/probe-log-cleanup", () => ({ stopEndpointProbeLogCleanup: () => {}, })); @@ -206,6 +217,9 @@ describe.sequential("lifecycle/shutdown", () => { vi.doMock("@/lib/public-status/scheduler", () => ({ stopPublicStatusRebuildScheduler: async () => {}, })); + vi.doMock("@/lib/availability/projection-worker", () => ({ + stopAvailabilityProjectionWorker: async () => {}, + })); vi.doMock("@/lib/provider-endpoints/probe-log-cleanup", () => ({ stopEndpointProbeLogCleanup: () => {}, })); @@ -289,6 +303,9 @@ describe.sequential("lifecycle/shutdown", () => { vi.doMock("@/lib/public-status/scheduler", () => ({ stopPublicStatusRebuildScheduler: async () => {}, })); + vi.doMock("@/lib/availability/projection-worker", () => ({ + stopAvailabilityProjectionWorker: async () => {}, + })); vi.doMock("@/lib/provider-endpoints/probe-log-cleanup", () => ({ stopEndpointProbeLogCleanup: async () => {}, })); @@ -331,6 +348,9 @@ describe.sequential("lifecycle/shutdown", () => { vi.doMock("@/lib/public-status/scheduler", () => ({ stopPublicStatusRebuildScheduler: async () => {}, })); + vi.doMock("@/lib/availability/projection-worker", () => ({ + stopAvailabilityProjectionWorker: async () => {}, + })); vi.doMock("@/lib/provider-endpoints/probe-log-cleanup", () => ({ stopEndpointProbeLogCleanup: () => {}, })); diff --git a/tests/unit/server-shutdown.test.ts b/tests/unit/server-shutdown.test.ts index caa628d97..f2aae86b3 100644 --- a/tests/unit/server-shutdown.test.ts +++ b/tests/unit/server-shutdown.test.ts @@ -258,6 +258,9 @@ describe.sequential("registerOrchestratedShutdown", () => { vi.doMock("@/lib/public-status/scheduler", () => ({ stopPublicStatusRebuildScheduler: async () => {}, })); + vi.doMock("@/lib/availability/projection-worker", () => ({ + stopAvailabilityProjectionWorker: async () => {}, + })); vi.doMock("@/lib/provider-endpoints/probe-log-cleanup", () => ({ stopEndpointProbeLogCleanup: () => {}, })); From c8a2e52462b50589b48f93fc5b6af38a6c4b227b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E4=B8=BA?= <43251607+Syh1906@users.noreply.github.com> Date: Wed, 12 Aug 2026 15:49:40 +0800 Subject: [PATCH 10/12] fix(dashboard): disable session message link prefetch (#1412) Co-authored-by: GPT-5 --- .../_components/error-details-dialog.test.tsx | 19 +++++++++++++++++-- .../components/SummaryTab.tsx | 2 +- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/app/[locale]/dashboard/logs/_components/error-details-dialog.test.tsx b/src/app/[locale]/dashboard/logs/_components/error-details-dialog.test.tsx index dc6f5deff..1acb562a4 100644 --- a/src/app/[locale]/dashboard/logs/_components/error-details-dialog.test.tsx +++ b/src/app/[locale]/dashboard/logs/_components/error-details-dialog.test.tsx @@ -36,8 +36,18 @@ beforeEach(() => { }); vi.mock("@/i18n/routing", () => ({ - Link: ({ href, children }: { href: string; children: ReactNode }) => ( - {children} + Link: ({ + href, + children, + prefetch, + }: { + href: string; + children: ReactNode; + prefetch?: boolean; + }) => ( + + {children} + ), })); @@ -497,6 +507,11 @@ describe("error-details-dialog layout", () => { expect(container.querySelector('a[href*="sessionId=pfx%3Ascope%3Aroot"]')).toBeTruthy(); expect(container.querySelector('a[href*="sessionId=physical-a"]')).toBeTruthy(); expect(container.querySelector('a[href*="requestId=203"]')).toBeTruthy(); + expect( + container + .querySelector('a[href*="/dashboard/sessions/pfx%3Ascope%3Aroot/messages"]') + ?.getAttribute("data-prefetch") + ).toBe("false"); expect(container.textContent).toContain("Canonical Session ID: pfx:scope:root"); expect(container.textContent).toContain("Client Session ID: physical-a"); unmount(); diff --git a/src/app/[locale]/dashboard/logs/_components/error-details-dialog/components/SummaryTab.tsx b/src/app/[locale]/dashboard/logs/_components/error-details-dialog/components/SummaryTab.tsx index b20ea4a2f..ad967e6e1 100644 --- a/src/app/[locale]/dashboard/logs/_components/error-details-dialog/components/SummaryTab.tsx +++ b/src/app/[locale]/dashboard/logs/_components/error-details-dialog/components/SummaryTab.tsx @@ -361,7 +361,7 @@ export function SummaryTab({
{identity.value === sessionId && hasMessages && !checkingMessages && ( - +