From 8403236330fd03196a10e60ecaf027c6075219dc Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Tue, 21 Jul 2026 05:44:49 -0400 Subject: [PATCH 01/12] feat(observability): add request-level Discovery routing trace --- drizzle/0111_happy_mauler.sql | 41 + drizzle/meta/0111_snapshot.json | 4697 +++++++++++++++++ drizzle/meta/_journal.json | 7 + messages/en/dashboard.json | 94 + messages/ja/dashboard.json | 94 + messages/ru/dashboard.json | 94 + messages/zh-CN/dashboard.json | 94 + messages/zh-TW/dashboard.json | 94 + src/actions/usage-logs.ts | 1 + .../_components/error-details-dialog.test.tsx | 249 + .../components/DiscoveryTraceView.tsx | 649 +++ .../components/LogicTraceTab.tsx | 15 + .../error-details-dialog/components/index.ts | 1 + .../error-details-dialog/index.tsx | 4 + .../_components/error-details-dialog/types.ts | 3 + .../provider-chain-popover.test.tsx | 179 + .../_components/provider-chain-popover.tsx | 178 + .../logs/_components/usage-logs-table.tsx | 2 + .../_components/virtualized-logs-table.tsx | 2 + src/app/v1/_lib/proxy/error-handler.ts | 2 + src/app/v1/_lib/proxy/forwarder.ts | 408 +- src/app/v1/_lib/proxy/response-handler.ts | 119 +- src/app/v1/_lib/proxy/session.ts | 256 +- src/drizzle/schema.ts | 4 + src/lib/ledger-backfill/trigger.sql | 32 +- src/lib/observability/discovery-metrics.ts | 32 +- .../redis/live-chain-store.storage.test.ts | 219 + src/lib/redis/live-chain-store.test.ts | 125 + src/lib/redis/live-chain-store.ts | 119 +- src/repository/_shared/transformers.ts | 2 + src/repository/message-write-buffer.ts | 12 +- src/repository/message.ts | 43 + src/repository/usage-logs.ts | 18 +- src/types/message.ts | 7 + src/types/routing-trace.ts | 299 ++ .../proxy-forwarder-hedge-first-byte.test.ts | 7 +- ...handler-endpoint-circuit-isolation.test.ts | 65 + tests/unit/proxy/routing-trace.test.ts | 353 ++ .../proxy/terminal-outcome-contract.test.ts | 2 + .../message-terminal-write-apis.test.ts | 26 + .../repository/message-write-buffer.test.ts | 40 + tests/unit/types/routing-trace.test.ts | 107 + tests/unit/usage-ledger/trigger.test.ts | 5 + 43 files changed, 8708 insertions(+), 92 deletions(-) create mode 100644 drizzle/0111_happy_mauler.sql create mode 100644 drizzle/meta/0111_snapshot.json create mode 100644 src/app/[locale]/dashboard/logs/_components/error-details-dialog/components/DiscoveryTraceView.tsx create mode 100644 src/lib/redis/live-chain-store.storage.test.ts create mode 100644 src/types/routing-trace.ts create mode 100644 tests/unit/proxy/routing-trace.test.ts create mode 100644 tests/unit/types/routing-trace.test.ts diff --git a/drizzle/0111_happy_mauler.sql b/drizzle/0111_happy_mauler.sql new file mode 100644 index 000000000..b0a595652 --- /dev/null +++ b/drizzle/0111_happy_mauler.sql @@ -0,0 +1,41 @@ +ALTER TABLE "message_request" ADD COLUMN "routing_trace" jsonb; + +-- Routing trace finalization is observability-only. Restrict the ledger trigger +-- to columns that can actually change its projection so trace-only patches do +-- not rewrite accounting rows. +DROP TRIGGER IF EXISTS trg_upsert_usage_ledger ON message_request; + +CREATE TRIGGER trg_upsert_usage_ledger +AFTER INSERT OR UPDATE OF + blocked_by, + status_code, + error_message, + provider_chain, + actual_response_model, + endpoint, + provider_id, + user_id, + "key", + model, + original_model, + api_type, + session_id, + cost_usd, + cost_multiplier, + group_cost_multiplier, + input_tokens, + output_tokens, + cache_creation_input_tokens, + cache_read_input_tokens, + cache_creation_5m_input_tokens, + cache_creation_1h_input_tokens, + cache_ttl_applied, + context_1m_applied, + swap_cache_ttl_applied, + duration_ms, + ttfb_ms, + client_ip, + created_at +ON message_request +FOR EACH ROW +EXECUTE FUNCTION fn_upsert_usage_ledger(); diff --git a/drizzle/meta/0111_snapshot.json b/drizzle/meta/0111_snapshot.json new file mode 100644 index 000000000..2c6d18c62 --- /dev/null +++ b/drizzle/meta/0111_snapshot.json @@ -0,0 +1,4697 @@ +{ + "id": "b59029b9-eae7-4d23-97b8-323ee78c8e99", + "prevId": "b6b7996d-5b70-4a31-a1c7-3a318b3398a0", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.audit_log": { + "name": "audit_log", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "action_category": { + "name": "action_category", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true + }, + "action_type": { + "name": "action_type", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true + }, + "target_type": { + "name": "target_type", + "type": "varchar(32)", + "primaryKey": false, + "notNull": false + }, + "target_id": { + "name": "target_id", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "target_name": { + "name": "target_name", + "type": "varchar(256)", + "primaryKey": false, + "notNull": false + }, + "before_value": { + "name": "before_value", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "after_value": { + "name": "after_value", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "operator_user_id": { + "name": "operator_user_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "operator_user_name": { + "name": "operator_user_name", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false + }, + "operator_key_id": { + "name": "operator_key_id", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "operator_key_name": { + "name": "operator_key_name", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false + }, + "operator_ip": { + "name": "operator_ip", + "type": "varchar(45)", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false + }, + "success": { + "name": "success", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "idx_audit_log_category_created_at": { + "name": "idx_audit_log_category_created_at", + "columns": [ + { + "expression": "action_category", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_audit_log_operator_user_created_at": { + "name": "idx_audit_log_operator_user_created_at", + "columns": [ + { + "expression": "operator_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"audit_log\".\"operator_user_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_audit_log_operator_ip_created_at": { + "name": "idx_audit_log_operator_ip_created_at", + "columns": [ + { + "expression": "operator_ip", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"audit_log\".\"operator_ip\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_audit_log_target": { + "name": "idx_audit_log_target", + "columns": [ + { + "expression": "target_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"audit_log\".\"target_type\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_audit_log_created_at_id": { + "name": "idx_audit_log_created_at_id", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.cloud_pricing_catalog": { + "name": "cloud_pricing_catalog", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "version": { + "name": "version", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true + }, + "currency": { + "name": "currency", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "default": "'USD'" + }, + "refreshed_at": { + "name": "refreshed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "providers": { + "name": "providers", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "vendors": { + "name": "vendors", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "model_count": { + "name": "model_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "synced_at": { + "name": "synced_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.error_rules": { + "name": "error_rules", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "pattern": { + "name": "pattern", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "match_type": { + "name": "match_type", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'regex'" + }, + "category": { + "name": "category", + "type": "varchar(50)", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "override_response": { + "name": "override_response", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "override_status_code": { + "name": "override_status_code", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "is_enabled": { + "name": "is_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "is_default": { + "name": "is_default", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "priority": { + "name": "priority", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "idx_error_rules_enabled": { + "name": "idx_error_rules_enabled", + "columns": [ + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "priority", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "unique_pattern": { + "name": "unique_pattern", + "columns": [ + { + "expression": "pattern", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_category": { + "name": "idx_category", + "columns": [ + { + "expression": "category", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_match_type": { + "name": "idx_match_type", + "columns": [ + { + "expression": "match_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.keys": { + "name": "keys", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "is_enabled": { + "name": "is_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "can_login_web_ui": { + "name": "can_login_web_ui", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "limit_5h_usd": { + "name": "limit_5h_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "limit_5h_reset_mode": { + "name": "limit_5h_reset_mode", + "type": "daily_reset_mode", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'rolling'" + }, + "limit_daily_usd": { + "name": "limit_daily_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "daily_reset_mode": { + "name": "daily_reset_mode", + "type": "daily_reset_mode", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'fixed'" + }, + "daily_reset_time": { + "name": "daily_reset_time", + "type": "varchar(5)", + "primaryKey": false, + "notNull": true, + "default": "'00:00'" + }, + "limit_weekly_usd": { + "name": "limit_weekly_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "limit_monthly_usd": { + "name": "limit_monthly_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "limit_total_usd": { + "name": "limit_total_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "cost_reset_at": { + "name": "cost_reset_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "limit_concurrent_sessions": { + "name": "limit_concurrent_sessions", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "provider_group": { + "name": "provider_group", + "type": "varchar(200)", + "primaryKey": false, + "notNull": false, + "default": "'default'" + }, + "cache_ttl_preference": { + "name": "cache_ttl_preference", + "type": "varchar(10)", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "idx_keys_user_id": { + "name": "idx_keys_user_id", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_keys_key": { + "name": "idx_keys_key", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_keys_created_at": { + "name": "idx_keys_created_at", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_keys_deleted_at": { + "name": "idx_keys_deleted_at", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.message_request": { + "name": "message_request", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "model": { + "name": "model", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "cost_usd": { + "name": "cost_usd", + "type": "numeric(21, 15)", + "primaryKey": false, + "notNull": false, + "default": "'0'" + }, + "cost_multiplier": { + "name": "cost_multiplier", + "type": "numeric(10, 4)", + "primaryKey": false, + "notNull": false + }, + "group_cost_multiplier": { + "name": "group_cost_multiplier", + "type": "numeric(10, 4)", + "primaryKey": false, + "notNull": false + }, + "cost_breakdown": { + "name": "cost_breakdown", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "session_id": { + "name": "session_id", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "request_sequence": { + "name": "request_sequence", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 1 + }, + "provider_chain": { + "name": "provider_chain", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "routing_trace": { + "name": "routing_trace", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "status_code": { + "name": "status_code", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "api_type": { + "name": "api_type", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "endpoint": { + "name": "endpoint", + "type": "varchar(256)", + "primaryKey": false, + "notNull": false + }, + "original_model": { + "name": "original_model", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false + }, + "actual_response_model": { + "name": "actual_response_model", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false + }, + "input_tokens": { + "name": "input_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "output_tokens": { + "name": "output_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "ttfb_ms": { + "name": "ttfb_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "cache_creation_input_tokens": { + "name": "cache_creation_input_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "cache_read_input_tokens": { + "name": "cache_read_input_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "cache_creation_5m_input_tokens": { + "name": "cache_creation_5m_input_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "cache_creation_1h_input_tokens": { + "name": "cache_creation_1h_input_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "cache_ttl_applied": { + "name": "cache_ttl_applied", + "type": "varchar(10)", + "primaryKey": false, + "notNull": false + }, + "context_1m_applied": { + "name": "context_1m_applied", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "swap_cache_ttl_applied": { + "name": "swap_cache_ttl_applied", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "special_settings": { + "name": "special_settings", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "hedge_losers": { + "name": "hedge_losers", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error_stack": { + "name": "error_stack", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error_cause": { + "name": "error_cause", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "blocked_by": { + "name": "blocked_by", + "type": "varchar(50)", + "primaryKey": false, + "notNull": false + }, + "blocked_reason": { + "name": "blocked_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false + }, + "client_ip": { + "name": "client_ip", + "type": "varchar(45)", + "primaryKey": false, + "notNull": false + }, + "messages_count": { + "name": "messages_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "idx_message_request_user_date_cost": { + "name": "idx_message_request_user_date_cost", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "cost_usd", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_user_created_at_cost_stats": { + "name": "idx_message_request_user_created_at_cost_stats", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "cost_usd", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND (\"message_request\".\"blocked_by\" IS NULL OR \"message_request\".\"blocked_by\" <> 'warmup')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_user_query": { + "name": "idx_message_request_user_query", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_provider_created_at_active": { + "name": "idx_message_request_provider_created_at_active", + "columns": [ + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND (\"message_request\".\"blocked_by\" IS NULL OR \"message_request\".\"blocked_by\" <> 'warmup')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_provider_created_at_finalized_active": { + "name": "idx_message_request_provider_created_at_finalized_active", + "columns": [ + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND \"message_request\".\"status_code\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_session_id": { + "name": "idx_message_request_session_id", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_session_id_prefix": { + "name": "idx_message_request_session_id_prefix", + "columns": [ + { + "expression": "\"session_id\" varchar_pattern_ops", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND (\"message_request\".\"blocked_by\" IS NULL OR \"message_request\".\"blocked_by\" <> 'warmup')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_session_seq": { + "name": "idx_message_request_session_seq", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "request_sequence", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_endpoint": { + "name": "idx_message_request_endpoint", + "columns": [ + { + "expression": "endpoint", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_blocked_by": { + "name": "idx_message_request_blocked_by", + "columns": [ + { + "expression": "blocked_by", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_provider_id": { + "name": "idx_message_request_provider_id", + "columns": [ + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_user_id": { + "name": "idx_message_request_user_id", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_key": { + "name": "idx_message_request_key", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_key_created_at_id": { + "name": "idx_message_request_key_created_at_id", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_key_model_active": { + "name": "idx_message_request_key_model_active", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "model", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND \"message_request\".\"model\" IS NOT NULL AND (\"message_request\".\"blocked_by\" IS NULL OR \"message_request\".\"blocked_by\" <> 'warmup')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_key_endpoint_active": { + "name": "idx_message_request_key_endpoint_active", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "endpoint", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND \"message_request\".\"endpoint\" IS NOT NULL AND (\"message_request\".\"blocked_by\" IS NULL OR \"message_request\".\"blocked_by\" <> 'warmup')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_created_at_id_active": { + "name": "idx_message_request_created_at_id_active", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_model_active": { + "name": "idx_message_request_model_active", + "columns": [ + { + "expression": "model", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND \"message_request\".\"model\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_status_code_active": { + "name": "idx_message_request_status_code_active", + "columns": [ + { + "expression": "status_code", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND \"message_request\".\"status_code\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_created_at": { + "name": "idx_message_request_created_at", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_deleted_at": { + "name": "idx_message_request_deleted_at", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_key_last_active": { + "name": "idx_message_request_key_last_active", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND (\"message_request\".\"blocked_by\" IS NULL OR \"message_request\".\"blocked_by\" <> 'warmup')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_key_cost_active": { + "name": "idx_message_request_key_cost_active", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "cost_usd", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND (\"message_request\".\"blocked_by\" IS NULL OR \"message_request\".\"blocked_by\" <> 'warmup')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_session_user_info": { + "name": "idx_message_request_session_user_info", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_message_request_client_ip_created_at": { + "name": "idx_message_request_client_ip_created_at", + "columns": [ + { + "expression": "client_ip", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"message_request\".\"deleted_at\" IS NULL AND \"message_request\".\"client_ip\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.model_prices": { + "name": "model_prices", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "model_name": { + "name": "model_name", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "price_data": { + "name": "price_data", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'cloud'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "idx_model_prices_latest": { + "name": "idx_model_prices_latest", + "columns": [ + { + "expression": "model_name", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_model_prices_model_name": { + "name": "idx_model_prices_model_name", + "columns": [ + { + "expression": "model_name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_model_prices_created_at": { + "name": "idx_model_prices_created_at", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_model_prices_source": { + "name": "idx_model_prices_source", + "columns": [ + { + "expression": "source", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_model_prices_vendor": { + "name": "idx_model_prices_vendor", + "columns": [ + { + "expression": "((\"price_data\" ->> 'vendor'))", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_model_prices_aliases": { + "name": "idx_model_prices_aliases", + "columns": [ + { + "expression": "((\"price_data\" -> 'aliases'))", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.notification_settings": { + "name": "notification_settings", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "use_legacy_mode": { + "name": "use_legacy_mode", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "circuit_breaker_enabled": { + "name": "circuit_breaker_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "circuit_breaker_webhook": { + "name": "circuit_breaker_webhook", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false + }, + "daily_leaderboard_enabled": { + "name": "daily_leaderboard_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "daily_leaderboard_webhook": { + "name": "daily_leaderboard_webhook", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false + }, + "daily_leaderboard_time": { + "name": "daily_leaderboard_time", + "type": "varchar(10)", + "primaryKey": false, + "notNull": false, + "default": "'09:00'" + }, + "daily_leaderboard_top_n": { + "name": "daily_leaderboard_top_n", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 5 + }, + "cost_alert_enabled": { + "name": "cost_alert_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "cost_alert_webhook": { + "name": "cost_alert_webhook", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false + }, + "cost_alert_threshold": { + "name": "cost_alert_threshold", + "type": "numeric(5, 2)", + "primaryKey": false, + "notNull": false, + "default": "'0.80'" + }, + "cost_alert_check_interval": { + "name": "cost_alert_check_interval", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 60 + }, + "cache_hit_rate_alert_enabled": { + "name": "cache_hit_rate_alert_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "cache_hit_rate_alert_webhook": { + "name": "cache_hit_rate_alert_webhook", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false + }, + "cache_hit_rate_alert_window_mode": { + "name": "cache_hit_rate_alert_window_mode", + "type": "varchar(10)", + "primaryKey": false, + "notNull": false, + "default": "'auto'" + }, + "cache_hit_rate_alert_check_interval": { + "name": "cache_hit_rate_alert_check_interval", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 5 + }, + "cache_hit_rate_alert_historical_lookback_days": { + "name": "cache_hit_rate_alert_historical_lookback_days", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 7 + }, + "cache_hit_rate_alert_min_eligible_requests": { + "name": "cache_hit_rate_alert_min_eligible_requests", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 20 + }, + "cache_hit_rate_alert_min_eligible_tokens": { + "name": "cache_hit_rate_alert_min_eligible_tokens", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "cache_hit_rate_alert_abs_min": { + "name": "cache_hit_rate_alert_abs_min", + "type": "numeric(5, 4)", + "primaryKey": false, + "notNull": false, + "default": "'0.05'" + }, + "cache_hit_rate_alert_drop_rel": { + "name": "cache_hit_rate_alert_drop_rel", + "type": "numeric(5, 4)", + "primaryKey": false, + "notNull": false, + "default": "'0.3'" + }, + "cache_hit_rate_alert_drop_abs": { + "name": "cache_hit_rate_alert_drop_abs", + "type": "numeric(5, 4)", + "primaryKey": false, + "notNull": false, + "default": "'0.1'" + }, + "cache_hit_rate_alert_cooldown_minutes": { + "name": "cache_hit_rate_alert_cooldown_minutes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 30 + }, + "cache_hit_rate_alert_top_n": { + "name": "cache_hit_rate_alert_top_n", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 10 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.notification_target_bindings": { + "name": "notification_target_bindings", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "notification_type": { + "name": "notification_type", + "type": "notification_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "target_id": { + "name": "target_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "is_enabled": { + "name": "is_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "schedule_cron": { + "name": "schedule_cron", + "type": "varchar(100)", + "primaryKey": false, + "notNull": false + }, + "schedule_timezone": { + "name": "schedule_timezone", + "type": "varchar(50)", + "primaryKey": false, + "notNull": false + }, + "template_override": { + "name": "template_override", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "unique_notification_target_binding": { + "name": "unique_notification_target_binding", + "columns": [ + { + "expression": "notification_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "target_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_notification_bindings_type": { + "name": "idx_notification_bindings_type", + "columns": [ + { + "expression": "notification_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_notification_bindings_target": { + "name": "idx_notification_bindings_target", + "columns": [ + { + "expression": "target_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "notification_target_bindings_target_id_webhook_targets_id_fk": { + "name": "notification_target_bindings_target_id_webhook_targets_id_fk", + "tableFrom": "notification_target_bindings", + "tableTo": "webhook_targets", + "columnsFrom": [ + "target_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.provider_endpoint_probe_logs": { + "name": "provider_endpoint_probe_logs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "endpoint_id": { + "name": "endpoint_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'scheduled'" + }, + "ok": { + "name": "ok", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "status_code": { + "name": "status_code", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "latency_ms": { + "name": "latency_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "error_type": { + "name": "error_type", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "idx_provider_endpoint_probe_logs_endpoint_created_at": { + "name": "idx_provider_endpoint_probe_logs_endpoint_created_at", + "columns": [ + { + "expression": "endpoint_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_provider_endpoint_probe_logs_created_at": { + "name": "idx_provider_endpoint_probe_logs_created_at", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "provider_endpoint_probe_logs_endpoint_id_provider_endpoints_id_fk": { + "name": "provider_endpoint_probe_logs_endpoint_id_provider_endpoints_id_fk", + "tableFrom": "provider_endpoint_probe_logs", + "tableTo": "provider_endpoints", + "columnsFrom": [ + "endpoint_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.provider_endpoints": { + "name": "provider_endpoints", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "vendor_id": { + "name": "vendor_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "provider_type": { + "name": "provider_type", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'claude'" + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "label": { + "name": "label", + "type": "varchar(200)", + "primaryKey": false, + "notNull": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "is_enabled": { + "name": "is_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "last_probed_at": { + "name": "last_probed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_probe_ok": { + "name": "last_probe_ok", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "last_probe_status_code": { + "name": "last_probe_status_code", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "last_probe_latency_ms": { + "name": "last_probe_latency_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "last_probe_error_type": { + "name": "last_probe_error_type", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "last_probe_error_message": { + "name": "last_probe_error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "uniq_provider_endpoints_vendor_type_url": { + "name": "uniq_provider_endpoints_vendor_type_url", + "columns": [ + { + "expression": "vendor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "url", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"provider_endpoints\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_provider_endpoints_vendor_type": { + "name": "idx_provider_endpoints_vendor_type", + "columns": [ + { + "expression": "vendor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"provider_endpoints\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_provider_endpoints_enabled": { + "name": "idx_provider_endpoints_enabled", + "columns": [ + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "vendor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"provider_endpoints\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_provider_endpoints_pick_enabled": { + "name": "idx_provider_endpoints_pick_enabled", + "columns": [ + { + "expression": "vendor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "sort_order", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"provider_endpoints\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_provider_endpoints_created_at": { + "name": "idx_provider_endpoints_created_at", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_provider_endpoints_deleted_at": { + "name": "idx_provider_endpoints_deleted_at", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "provider_endpoints_vendor_id_provider_vendors_id_fk": { + "name": "provider_endpoints_vendor_id_provider_vendors_id_fk", + "tableFrom": "provider_endpoints", + "tableTo": "provider_vendors", + "columnsFrom": [ + "vendor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.provider_groups": { + "name": "provider_groups", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(200)", + "primaryKey": false, + "notNull": true + }, + "cost_multiplier": { + "name": "cost_multiplier", + "type": "numeric(10, 4)", + "primaryKey": false, + "notNull": true, + "default": "'1.0'" + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "provider_groups_name_unique": { + "name": "provider_groups_name_unique", + "nullsNotDistinct": false, + "columns": [ + "name" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.provider_vendors": { + "name": "provider_vendors", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "website_domain": { + "name": "website_domain", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "varchar(200)", + "primaryKey": false, + "notNull": false + }, + "website_url": { + "name": "website_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "favicon_url": { + "name": "favicon_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "uniq_provider_vendors_website_domain": { + "name": "uniq_provider_vendors_website_domain", + "columns": [ + { + "expression": "website_domain", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_provider_vendors_created_at": { + "name": "idx_provider_vendors_created_at", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.providers": { + "name": "providers", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "url": { + "name": "url", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "provider_vendor_id": { + "name": "provider_vendor_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "is_enabled": { + "name": "is_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "weight": { + "name": "weight", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "priority": { + "name": "priority", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "group_priorities": { + "name": "group_priorities", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'null'::jsonb" + }, + "cost_multiplier": { + "name": "cost_multiplier", + "type": "numeric(10, 4)", + "primaryKey": false, + "notNull": false, + "default": "'1.0'" + }, + "group_tag": { + "name": "group_tag", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false + }, + "provider_type": { + "name": "provider_type", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'claude'" + }, + "preserve_client_ip": { + "name": "preserve_client_ip", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "disable_session_reuse": { + "name": "disable_session_reuse", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "model_redirects": { + "name": "model_redirects", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "allowed_models": { + "name": "allowed_models", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'null'::jsonb" + }, + "allowed_clients": { + "name": "allowed_clients", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "blocked_clients": { + "name": "blocked_clients", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "active_time_start": { + "name": "active_time_start", + "type": "varchar(5)", + "primaryKey": false, + "notNull": false + }, + "active_time_end": { + "name": "active_time_end", + "type": "varchar(5)", + "primaryKey": false, + "notNull": false + }, + "codex_instructions_strategy": { + "name": "codex_instructions_strategy", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false, + "default": "'auto'" + }, + "mcp_passthrough_type": { + "name": "mcp_passthrough_type", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'none'" + }, + "mcp_passthrough_url": { + "name": "mcp_passthrough_url", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false + }, + "limit_5h_usd": { + "name": "limit_5h_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "limit_5h_reset_mode": { + "name": "limit_5h_reset_mode", + "type": "daily_reset_mode", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'rolling'" + }, + "limit_daily_usd": { + "name": "limit_daily_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "daily_reset_mode": { + "name": "daily_reset_mode", + "type": "daily_reset_mode", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'fixed'" + }, + "daily_reset_time": { + "name": "daily_reset_time", + "type": "varchar(5)", + "primaryKey": false, + "notNull": true, + "default": "'00:00'" + }, + "limit_weekly_usd": { + "name": "limit_weekly_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "limit_monthly_usd": { + "name": "limit_monthly_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "limit_total_usd": { + "name": "limit_total_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "total_cost_reset_at": { + "name": "total_cost_reset_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "limit_concurrent_sessions": { + "name": "limit_concurrent_sessions", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "max_retry_attempts": { + "name": "max_retry_attempts", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "circuit_breaker_failure_threshold": { + "name": "circuit_breaker_failure_threshold", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 5 + }, + "circuit_breaker_open_duration": { + "name": "circuit_breaker_open_duration", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 1800000 + }, + "circuit_breaker_half_open_success_threshold": { + "name": "circuit_breaker_half_open_success_threshold", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 2 + }, + "proxy_url": { + "name": "proxy_url", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false + }, + "proxy_fallback_to_direct": { + "name": "proxy_fallback_to_direct", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "custom_headers": { + "name": "custom_headers", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "first_byte_timeout_streaming_ms": { + "name": "first_byte_timeout_streaming_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "streaming_idle_timeout_ms": { + "name": "streaming_idle_timeout_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "request_timeout_non_streaming_ms": { + "name": "request_timeout_non_streaming_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "website_url": { + "name": "website_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "favicon_url": { + "name": "favicon_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cache_ttl_preference": { + "name": "cache_ttl_preference", + "type": "varchar(10)", + "primaryKey": false, + "notNull": false + }, + "swap_cache_ttl_billing": { + "name": "swap_cache_ttl_billing", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "context_1m_preference": { + "name": "context_1m_preference", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "codex_reasoning_effort_preference": { + "name": "codex_reasoning_effort_preference", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "codex_reasoning_summary_preference": { + "name": "codex_reasoning_summary_preference", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "codex_text_verbosity_preference": { + "name": "codex_text_verbosity_preference", + "type": "varchar(10)", + "primaryKey": false, + "notNull": false + }, + "codex_parallel_tool_calls_preference": { + "name": "codex_parallel_tool_calls_preference", + "type": "varchar(10)", + "primaryKey": false, + "notNull": false + }, + "codex_image_generation_preference": { + "name": "codex_image_generation_preference", + "type": "varchar(10)", + "primaryKey": false, + "notNull": false + }, + "codex_service_tier_preference": { + "name": "codex_service_tier_preference", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "anthropic_max_tokens_preference": { + "name": "anthropic_max_tokens_preference", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "anthropic_thinking_budget_preference": { + "name": "anthropic_thinking_budget_preference", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "anthropic_adaptive_thinking": { + "name": "anthropic_adaptive_thinking", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'null'::jsonb" + }, + "gemini_google_search_preference": { + "name": "gemini_google_search_preference", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "tpm": { + "name": "tpm", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "rpm": { + "name": "rpm", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "rpd": { + "name": "rpd", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "cc": { + "name": "cc", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "idx_providers_enabled_priority": { + "name": "idx_providers_enabled_priority", + "columns": [ + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "priority", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "weight", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"providers\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_providers_group": { + "name": "idx_providers_group", + "columns": [ + { + "expression": "group_tag", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"providers\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_providers_vendor_type_url_active": { + "name": "idx_providers_vendor_type_url_active", + "columns": [ + { + "expression": "provider_vendor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "url", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"providers\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_providers_created_at": { + "name": "idx_providers_created_at", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_providers_deleted_at": { + "name": "idx_providers_deleted_at", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_providers_vendor_type": { + "name": "idx_providers_vendor_type", + "columns": [ + { + "expression": "provider_vendor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"providers\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_providers_enabled_vendor_type": { + "name": "idx_providers_enabled_vendor_type", + "columns": [ + { + "expression": "provider_vendor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"providers\".\"deleted_at\" IS NULL AND \"providers\".\"is_enabled\" = true AND \"providers\".\"provider_vendor_id\" IS NOT NULL AND \"providers\".\"provider_vendor_id\" > 0", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "providers_provider_vendor_id_provider_vendors_id_fk": { + "name": "providers_provider_vendor_id_provider_vendors_id_fk", + "tableFrom": "providers", + "tableTo": "provider_vendors", + "columnsFrom": [ + "provider_vendor_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.request_filters": { + "name": "request_filters", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(100)", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true + }, + "action": { + "name": "action", + "type": "varchar(30)", + "primaryKey": false, + "notNull": true + }, + "match_type": { + "name": "match_type", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "target": { + "name": "target", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "replacement": { + "name": "replacement", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "priority": { + "name": "priority", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "is_enabled": { + "name": "is_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "binding_type": { + "name": "binding_type", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'global'" + }, + "provider_ids": { + "name": "provider_ids", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "group_tags": { + "name": "group_tags", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "rule_mode": { + "name": "rule_mode", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'simple'" + }, + "execution_phase": { + "name": "execution_phase", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'guard'" + }, + "operations": { + "name": "operations", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "idx_request_filters_enabled": { + "name": "idx_request_filters_enabled", + "columns": [ + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "priority", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_request_filters_scope": { + "name": "idx_request_filters_scope", + "columns": [ + { + "expression": "scope", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_request_filters_action": { + "name": "idx_request_filters_action", + "columns": [ + { + "expression": "action", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_request_filters_binding": { + "name": "idx_request_filters_binding", + "columns": [ + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "binding_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_request_filters_phase": { + "name": "idx_request_filters_phase", + "columns": [ + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "execution_phase", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sensitive_words": { + "name": "sensitive_words", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "word": { + "name": "word", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "match_type": { + "name": "match_type", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'contains'" + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_enabled": { + "name": "is_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": { + "idx_sensitive_words_enabled": { + "name": "idx_sensitive_words_enabled", + "columns": [ + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "match_type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_sensitive_words_created_at": { + "name": "idx_sensitive_words_created_at", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.system_settings": { + "name": "system_settings", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "site_title": { + "name": "site_title", + "type": "varchar(128)", + "primaryKey": false, + "notNull": true, + "default": "'Claude Code Hub'" + }, + "allow_global_usage_view": { + "name": "allow_global_usage_view", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "currency_display": { + "name": "currency_display", + "type": "varchar(10)", + "primaryKey": false, + "notNull": true, + "default": "'USD'" + }, + "billing_model_source": { + "name": "billing_model_source", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'original'" + }, + "codex_priority_billing_source": { + "name": "codex_priority_billing_source", + "type": "varchar(20)", + "primaryKey": false, + "notNull": true, + "default": "'requested'" + }, + "bill_non_successful_requests": { + "name": "bill_non_successful_requests", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "bill_hedge_losers": { + "name": "bill_hedge_losers", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "discovery_enabled": { + "name": "discovery_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "discovery_concurrency": { + "name": "discovery_concurrency", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 2 + }, + "max_discovery_rounds": { + "name": "max_discovery_rounds", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 2 + }, + "discovery_sla_ms": { + "name": "discovery_sla_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 10000 + }, + "sticky_sla_ms": { + "name": "sticky_sla_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 20000 + }, + "racing_total_timeout_ms": { + "name": "racing_total_timeout_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 60000 + }, + "sticky_timeout_cooldown_ms": { + "name": "sticky_timeout_cooldown_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 300000 + }, + "timezone": { + "name": "timezone", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "enable_auto_cleanup": { + "name": "enable_auto_cleanup", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "cleanup_retention_days": { + "name": "cleanup_retention_days", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 30 + }, + "cleanup_schedule": { + "name": "cleanup_schedule", + "type": "varchar(50)", + "primaryKey": false, + "notNull": false, + "default": "'0 2 * * *'" + }, + "cleanup_batch_size": { + "name": "cleanup_batch_size", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 10000 + }, + "enable_client_version_check": { + "name": "enable_client_version_check", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "verbose_provider_error": { + "name": "verbose_provider_error", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "pass_through_upstream_error_message": { + "name": "pass_through_upstream_error_message", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "enable_http2": { + "name": "enable_http2", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "enable_openai_responses_websocket": { + "name": "enable_openai_responses_websocket", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "enable_high_concurrency_mode": { + "name": "enable_high_concurrency_mode", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "intercept_anthropic_warmup_requests": { + "name": "intercept_anthropic_warmup_requests", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "enable_thinking_signature_rectifier": { + "name": "enable_thinking_signature_rectifier", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "enable_thinking_budget_rectifier": { + "name": "enable_thinking_budget_rectifier", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "enable_thinking_effort_conflict_rectifier": { + "name": "enable_thinking_effort_conflict_rectifier", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "enable_gemini_function_id_rectifier": { + "name": "enable_gemini_function_id_rectifier", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "enable_billing_header_rectifier": { + "name": "enable_billing_header_rectifier", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "enable_response_input_rectifier": { + "name": "enable_response_input_rectifier", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "allow_non_conversation_endpoint_provider_fallback": { + "name": "allow_non_conversation_endpoint_provider_fallback", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "fake_streaming_whitelist": { + "name": "fake_streaming_whitelist", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "enable_codex_session_id_completion": { + "name": "enable_codex_session_id_completion", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "enable_claude_metadata_user_id_injection": { + "name": "enable_claude_metadata_user_id_injection", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "enable_response_fixer": { + "name": "enable_response_fixer", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "response_fixer_config": { + "name": "response_fixer_config", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'{\"fixTruncatedJson\":true,\"fixSseFormat\":true,\"fixEncoding\":true,\"maxJsonDepth\":200,\"maxFixSize\":1048576}'::jsonb" + }, + "quota_db_refresh_interval_seconds": { + "name": "quota_db_refresh_interval_seconds", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 10 + }, + "quota_lease_percent_5h": { + "name": "quota_lease_percent_5h", + "type": "numeric(5, 4)", + "primaryKey": false, + "notNull": false, + "default": "'0.05'" + }, + "quota_lease_percent_daily": { + "name": "quota_lease_percent_daily", + "type": "numeric(5, 4)", + "primaryKey": false, + "notNull": false, + "default": "'0.05'" + }, + "quota_lease_percent_weekly": { + "name": "quota_lease_percent_weekly", + "type": "numeric(5, 4)", + "primaryKey": false, + "notNull": false, + "default": "'0.05'" + }, + "quota_lease_percent_monthly": { + "name": "quota_lease_percent_monthly", + "type": "numeric(5, 4)", + "primaryKey": false, + "notNull": false, + "default": "'0.05'" + }, + "quota_lease_cap_usd": { + "name": "quota_lease_cap_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "ip_extraction_config": { + "name": "ip_extraction_config", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "ip_geo_lookup_enabled": { + "name": "ip_geo_lookup_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "public_status_window_hours": { + "name": "public_status_window_hours", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 24 + }, + "public_status_aggregation_interval_minutes": { + "name": "public_status_aggregation_interval_minutes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 5 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.usage_ledger": { + "name": "usage_ledger", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "request_id": { + "name": "request_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "final_provider_id": { + "name": "final_provider_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "model": { + "name": "model", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false + }, + "original_model": { + "name": "original_model", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false + }, + "actual_response_model": { + "name": "actual_response_model", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false + }, + "endpoint": { + "name": "endpoint", + "type": "varchar(256)", + "primaryKey": false, + "notNull": false + }, + "api_type": { + "name": "api_type", + "type": "varchar(20)", + "primaryKey": false, + "notNull": false + }, + "session_id": { + "name": "session_id", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "status_code": { + "name": "status_code", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "is_success": { + "name": "is_success", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "success_rate_outcome": { + "name": "success_rate_outcome", + "type": "varchar(16)", + "primaryKey": false, + "notNull": false + }, + "blocked_by": { + "name": "blocked_by", + "type": "varchar(50)", + "primaryKey": false, + "notNull": false + }, + "cost_usd": { + "name": "cost_usd", + "type": "numeric(21, 15)", + "primaryKey": false, + "notNull": false, + "default": "'0'" + }, + "cost_multiplier": { + "name": "cost_multiplier", + "type": "numeric(10, 4)", + "primaryKey": false, + "notNull": false + }, + "group_cost_multiplier": { + "name": "group_cost_multiplier", + "type": "numeric(10, 4)", + "primaryKey": false, + "notNull": false + }, + "input_tokens": { + "name": "input_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "output_tokens": { + "name": "output_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "cache_creation_input_tokens": { + "name": "cache_creation_input_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "cache_read_input_tokens": { + "name": "cache_read_input_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "cache_creation_5m_input_tokens": { + "name": "cache_creation_5m_input_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "cache_creation_1h_input_tokens": { + "name": "cache_creation_1h_input_tokens", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "cache_ttl_applied": { + "name": "cache_ttl_applied", + "type": "varchar(10)", + "primaryKey": false, + "notNull": false + }, + "context_1m_applied": { + "name": "context_1m_applied", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "swap_cache_ttl_applied": { + "name": "swap_cache_ttl_applied", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "ttfb_ms": { + "name": "ttfb_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "client_ip": { + "name": "client_ip", + "type": "varchar(45)", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "idx_usage_ledger_request_id": { + "name": "idx_usage_ledger_request_id", + "columns": [ + { + "expression": "request_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_user_created_at": { + "name": "idx_usage_ledger_user_created_at", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_ledger\".\"blocked_by\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_key_created_at": { + "name": "idx_usage_ledger_key_created_at", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_ledger\".\"blocked_by\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_provider_created_at": { + "name": "idx_usage_ledger_provider_created_at", + "columns": [ + { + "expression": "final_provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_ledger\".\"blocked_by\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_created_at_minute": { + "name": "idx_usage_ledger_created_at_minute", + "columns": [ + { + "expression": "date_trunc('minute', \"created_at\" AT TIME ZONE 'UTC')", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_created_at_desc_id": { + "name": "idx_usage_ledger_created_at_desc_id", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_session_id": { + "name": "idx_usage_ledger_session_id", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_ledger\".\"session_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_model": { + "name": "idx_usage_ledger_model", + "columns": [ + { + "expression": "model", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_ledger\".\"model\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_key_cost": { + "name": "idx_usage_ledger_key_cost", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "cost_usd", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "endpoint", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_ledger\".\"blocked_by\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_user_cost_cover": { + "name": "idx_usage_ledger_user_cost_cover", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "cost_usd", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "endpoint", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_ledger\".\"blocked_by\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_provider_cost_cover": { + "name": "idx_usage_ledger_provider_cost_cover", + "columns": [ + { + "expression": "final_provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "cost_usd", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "endpoint", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_ledger\".\"blocked_by\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_usage_ledger_key_created_at_desc_cover": { + "name": "idx_usage_ledger_key_created_at_desc_cover", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "\"created_at\" DESC NULLS LAST", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "final_provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"usage_ledger\".\"blocked_by\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.users": { + "name": "users", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "role": { + "name": "role", + "type": "varchar", + "primaryKey": false, + "notNull": false, + "default": "'user'" + }, + "rpm_limit": { + "name": "rpm_limit", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "daily_limit_usd": { + "name": "daily_limit_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "provider_group": { + "name": "provider_group", + "type": "varchar(200)", + "primaryKey": false, + "notNull": false, + "default": "'default'" + }, + "tags": { + "name": "tags", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'[]'::jsonb" + }, + "limit_5h_usd": { + "name": "limit_5h_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "limit_5h_reset_mode": { + "name": "limit_5h_reset_mode", + "type": "daily_reset_mode", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'rolling'" + }, + "limit_weekly_usd": { + "name": "limit_weekly_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "limit_monthly_usd": { + "name": "limit_monthly_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "limit_total_usd": { + "name": "limit_total_usd", + "type": "numeric(10, 2)", + "primaryKey": false, + "notNull": false + }, + "cost_reset_at": { + "name": "cost_reset_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "limit_5h_cost_reset_at": { + "name": "limit_5h_cost_reset_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "limit_concurrent_sessions": { + "name": "limit_concurrent_sessions", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "daily_reset_mode": { + "name": "daily_reset_mode", + "type": "daily_reset_mode", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'fixed'" + }, + "daily_reset_time": { + "name": "daily_reset_time", + "type": "varchar(5)", + "primaryKey": false, + "notNull": true, + "default": "'00:00'" + }, + "is_enabled": { + "name": "is_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "allowed_clients": { + "name": "allowed_clients", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'[]'::jsonb" + }, + "allowed_models": { + "name": "allowed_models", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'[]'::jsonb" + }, + "blocked_clients": { + "name": "blocked_clients", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "idx_users_active_role_sort": { + "name": "idx_users_active_role_sort", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "role", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"users\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_users_enabled_expires_at": { + "name": "idx_users_enabled_expires_at", + "columns": [ + { + "expression": "is_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"users\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_users_tags_gin": { + "name": "idx_users_tags_gin", + "columns": [ + { + "expression": "tags", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"users\".\"deleted_at\" IS NULL", + "concurrently": false, + "method": "gin", + "with": {} + }, + "idx_users_created_at": { + "name": "idx_users_created_at", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idx_users_deleted_at": { + "name": "idx_users_deleted_at", + "columns": [ + { + "expression": "deleted_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.webhook_targets": { + "name": "webhook_targets", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(100)", + "primaryKey": false, + "notNull": true + }, + "provider_type": { + "name": "provider_type", + "type": "webhook_provider_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "webhook_url": { + "name": "webhook_url", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": false + }, + "telegram_bot_token": { + "name": "telegram_bot_token", + "type": "varchar(256)", + "primaryKey": false, + "notNull": false + }, + "telegram_chat_id": { + "name": "telegram_chat_id", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "dingtalk_secret": { + "name": "dingtalk_secret", + "type": "varchar(256)", + "primaryKey": false, + "notNull": false + }, + "custom_template": { + "name": "custom_template", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "custom_headers": { + "name": "custom_headers", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "proxy_url": { + "name": "proxy_url", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false + }, + "proxy_fallback_to_direct": { + "name": "proxy_fallback_to_direct", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "is_enabled": { + "name": "is_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "last_test_at": { + "name": "last_test_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_test_result": { + "name": "last_test_result", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.daily_reset_mode": { + "name": "daily_reset_mode", + "schema": "public", + "values": [ + "fixed", + "rolling" + ] + }, + "public.notification_type": { + "name": "notification_type", + "schema": "public", + "values": [ + "circuit_breaker", + "daily_leaderboard", + "cost_alert", + "cache_hit_rate_alert" + ] + }, + "public.webhook_provider_type": { + "name": "webhook_provider_type", + "schema": "public", + "values": [ + "wechat", + "feishu", + "dingtalk", + "telegram", + "custom" + ] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index b4b89464d..cb5a1b368 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -771,6 +771,13 @@ "when": 1784571513591, "tag": "0110_daffy_rawhide_kid", "breakpoints": true + }, + { + "idx": 111, + "version": "7", + "when": 1784622924311, + "tag": "0111_happy_mauler", + "breakpoints": true } ] } diff --git a/messages/en/dashboard.json b/messages/en/dashboard.json index eb8ec252f..8422afa78 100644 --- a/messages/en/dashboard.json +++ b/messages/en/dashboard.json @@ -379,6 +379,100 @@ "overridden": "Overridden by provider", "tooltip": "Thinking effort requested by the client (output_config.effort), shown verbatim. The proxy does not rename or convert levels." }, + "routingTrace": { + "title": "Routing mode", + "modes": { + "discovery": "Bounded Discovery", + "legacy_hedge": "Legacy Hedge", + "legacy_serial": "Legacy serial fallback", + "single_upstream": "Single upstream" + }, + "bypassed": "Discovery was not used: {reason}", + "bypassReasons": { + "disabled": "Discovery is disabled", + "non_streaming": "Non-streaming request", + "retry_not_allowed": "This endpoint does not allow retries", + "provider_switch_not_allowed": "This endpoint does not allow provider switching", + "raw_passthrough": "Raw passthrough request", + "unsupported_protocol": "Protocol is not supported by Discovery", + "websocket": "WebSocket request", + "streaming_hedge_disabled": "Streaming racing is disabled for this request", + "raw_cross_provider_fallback": "Raw cross-provider fallback is enabled", + "missing_session": "Session identity is missing", + "missing_key": "API key identity is missing", + "rollout_ineligible": "Request is outside the current rollout", + "redis_capability_unavailable": "Redis binding capability is unavailable", + "binding_conflict": "Session binding conflict", + "lease_conflict": "Another request owns the Discovery lease", + "lease_unavailable": "Discovery lease is unavailable", + "unknown": "Eligibility requirement was not met" + }, + "discoveryTitle": "Discovery rounds", + "discoveryCompact": "{rounds}R · {attempts} tries", + "roundsLabel": "Rounds", + "attemptsLabel": "Attempts", + "winnerOrigin": "Winner origin", + "viewDetails": "View Discovery details", + "stickyPhase": "Sticky probe", + "round": "Round {round}", + "attempts": "{count} attempts", + "noAttempts": "No provider attempts were recorded", + "providerFallback": "Provider {id}", + "elapsed": "+{elapsed}ms", + "priority": "Priority {priority}", + "roles": { + "sticky": "Sticky", + "normal": "Candidate", + "fallback": "Fallback" + }, + "outcomes": { + "success": "Success", + "pending": "Pending", + "ready": "Ready", + "held": "Ready, held", + "winner": "Winner", + "failed": "Failed", + "cancelled": "Cancelled", + "timeout": "SLA timeout", + "client_abort": "Client aborted", + "deadline": "Total deadline" + }, + "terminalOutcome": "Request result", + "bindingResult": "Sticky binding", + "bindingActions": { + "create": "Create", + "renew": "Renew", + "clear": "Clear", + "none": "No change" + }, + "bindingOutcomes": { + "updated": "Updated", + "cleared": "Cleared", + "skipped": "Skipped", + "failed": "Failed", + "unknown": "Unknown" + }, + "fallbackPromoted": "Promoted to fallback", + "winnerCommitted": "Winner committed", + "roundStarted": "Round started", + "events": { + "started": "Request started", + "ready": "Valid first content received", + "held": "Delivery held by routing policy", + "fallbackPromoted": "Promoted to fallback", + "winnerCommitted": "Committed as winner", + "finished": "Finished: {outcome}" + }, + "traceTruncated": "Some events were omitted because this trace reached its storage limit.", + "summary": "{rounds} rounds · {attempts} attempts · max {maxActive} active", + "config": "Configuration snapshot", + "configConcurrency": "Concurrency", + "configMaxRounds": "Maximum rounds", + "configDiscoverySla": "Round SLA", + "configStickySla": "Sticky SLA", + "configTotalTimeout": "Total timeout", + "configStickyCooldown": "Sticky timeout cooldown" + }, "logicTrace": { "title": "Decision Chain", "noDecisionData": "No decision data available", diff --git a/messages/ja/dashboard.json b/messages/ja/dashboard.json index 921d03678..3b6be2ece 100644 --- a/messages/ja/dashboard.json +++ b/messages/ja/dashboard.json @@ -379,6 +379,100 @@ "overridden": "プロバイダーにより上書き", "tooltip": "クライアントがリクエストボディで指定した思考強度 (output_config.effort)。プロキシは値をそのまま表示し、レベル名を変換しません。" }, + "routingTrace": { + "title": "ルーティングモード", + "modes": { + "discovery": "限定 Discovery", + "legacy_hedge": "従来の Hedge 競争", + "legacy_serial": "従来の直列フェイルオーバー", + "single_upstream": "単一アップストリーム" + }, + "bypassed": "このリクエストでは Discovery を使用していません：{reason}", + "bypassReasons": { + "disabled": "Discovery が無効です", + "non_streaming": "非ストリーミングリクエストです", + "retry_not_allowed": "このエンドポイントでは再試行できません", + "provider_switch_not_allowed": "このエンドポイントではプロバイダーを切り替えられません", + "raw_passthrough": "Raw パススルーリクエストです", + "unsupported_protocol": "Discovery が対応していないプロトコルです", + "websocket": "WebSocket リクエストです", + "streaming_hedge_disabled": "このリクエストではストリーミング競争が無効です", + "raw_cross_provider_fallback": "Raw クロスプロバイダーフォールバックが有効です", + "missing_session": "Session 識別子がありません", + "missing_key": "API Key 識別子がありません", + "rollout_ineligible": "現在のロールアウト対象外です", + "redis_capability_unavailable": "Redis バインディング機能を利用できません", + "binding_conflict": "Session バインディングが競合しています", + "lease_conflict": "別のリクエストが Discovery リースを保持しています", + "lease_unavailable": "Discovery リースを利用できません", + "unknown": "Discovery の適格条件を満たしていません" + }, + "discoveryTitle": "Discovery ラウンド", + "discoveryCompact": "{rounds}R · {attempts} 回", + "roundsLabel": "ラウンド", + "attemptsLabel": "試行", + "winnerOrigin": "勝者の経路", + "viewDetails": "Discovery の詳細を表示", + "stickyPhase": "Sticky プローブ", + "round": "ラウンド {round}", + "attempts": "{count} 回の試行", + "noAttempts": "プロバイダー試行の記録がありません", + "providerFallback": "プロバイダー {id}", + "elapsed": "+{elapsed}ms", + "priority": "優先度 {priority}", + "roles": { + "sticky": "Sticky", + "normal": "候補", + "fallback": "フォールバック" + }, + "outcomes": { + "success": "正常終了", + "pending": "待機中", + "ready": "準備完了", + "held": "準備完了、保留中", + "winner": "勝者", + "failed": "失敗", + "cancelled": "キャンセル済み", + "timeout": "SLA タイムアウト", + "client_abort": "クライアント切断", + "deadline": "全体期限" + }, + "terminalOutcome": "リクエスト結果", + "bindingResult": "Sticky バインド", + "bindingActions": { + "create": "作成", + "renew": "更新", + "clear": "解除", + "none": "変更なし" + }, + "bindingOutcomes": { + "updated": "更新済み", + "cleared": "解除済み", + "skipped": "スキップ", + "failed": "失敗", + "unknown": "不明" + }, + "fallbackPromoted": "フォールバックに昇格", + "winnerCommitted": "勝者を確定", + "roundStarted": "ラウンド開始", + "events": { + "started": "リクエスト開始", + "ready": "有効な先頭データを受信", + "held": "ルーティング規則により配信を保留", + "fallbackPromoted": "フォールバックに昇格", + "winnerCommitted": "勝者として確定", + "finished": "完了：{outcome}" + }, + "traceTruncated": "保存上限に達したため、一部のイベントは省略されました。", + "summary": "{rounds} ラウンド · {attempts} 試行 · 最大同時実行 {maxActive}", + "config": "設定スナップショット", + "configConcurrency": "同時実行数", + "configMaxRounds": "最大ラウンド数", + "configDiscoverySla": "ラウンド SLA", + "configStickySla": "Sticky SLA", + "configTotalTimeout": "全体タイムアウト", + "configStickyCooldown": "Sticky タイムアウトのクールダウン" + }, "logicTrace": { "title": "決定チェーン", "noDecisionData": "決定データがありません", diff --git a/messages/ru/dashboard.json b/messages/ru/dashboard.json index 5f5ad60dc..1b1f66545 100644 --- a/messages/ru/dashboard.json +++ b/messages/ru/dashboard.json @@ -379,6 +379,100 @@ "overridden": "Переопределено провайдером", "tooltip": "Уровень усилий на размышления, запрошенный клиентом (output_config.effort); показан как есть — прокси не переименовывает и не преобразует уровни." }, + "routingTrace": { + "title": "Режим маршрутизации", + "modes": { + "discovery": "Ограниченный Discovery", + "legacy_hedge": "Прежняя гонка Hedge", + "legacy_serial": "Прежнее последовательное переключение", + "single_upstream": "Один upstream" + }, + "bypassed": "Discovery не использован: {reason}", + "bypassReasons": { + "disabled": "Discovery отключён", + "non_streaming": "Непотоковый запрос", + "retry_not_allowed": "Эта конечная точка не разрешает повторы", + "provider_switch_not_allowed": "Эта конечная точка не разрешает смену провайдера", + "raw_passthrough": "Запрос с прямой передачей данных", + "unsupported_protocol": "Протокол не поддерживается Discovery", + "websocket": "Запрос WebSocket", + "streaming_hedge_disabled": "Потоковая гонка отключена для этого запроса", + "raw_cross_provider_fallback": "Включено прямое переключение между провайдерами", + "missing_session": "Нет идентификатора Session", + "missing_key": "Нет идентификатора API Key", + "rollout_ineligible": "Запрос не входит в текущий rollout", + "redis_capability_unavailable": "Возможность привязки Redis недоступна", + "binding_conflict": "Конфликт привязки Session", + "lease_conflict": "Lease Discovery занят другим запросом", + "lease_unavailable": "Lease Discovery недоступен", + "unknown": "Условия запуска Discovery не выполнены" + }, + "discoveryTitle": "Раунды Discovery", + "discoveryCompact": "{rounds} р. · {attempts} поп.", + "roundsLabel": "Раунды", + "attemptsLabel": "Попытки", + "winnerOrigin": "Источник победителя", + "viewDetails": "Открыть детали Discovery", + "stickyPhase": "Проверка Sticky", + "round": "Раунд {round}", + "attempts": "Попыток: {count}", + "noAttempts": "Попытки провайдеров не записаны", + "providerFallback": "Провайдер {id}", + "elapsed": "+{elapsed} мс", + "priority": "Приоритет {priority}", + "roles": { + "sticky": "Sticky", + "normal": "Кандидат", + "fallback": "Резерв" + }, + "outcomes": { + "success": "Успешно", + "pending": "Ожидание", + "ready": "Готов", + "held": "Готов, удерживается", + "winner": "Победитель", + "failed": "Ошибка", + "cancelled": "Отменён", + "timeout": "Тайм-аут SLA", + "client_abort": "Клиент отключился", + "deadline": "Общий срок" + }, + "terminalOutcome": "Результат запроса", + "bindingResult": "Sticky-привязка", + "bindingActions": { + "create": "Создать", + "renew": "Продлить", + "clear": "Очистить", + "none": "Без изменений" + }, + "bindingOutcomes": { + "updated": "Обновлено", + "cleared": "Очищено", + "skipped": "Пропущено", + "failed": "Ошибка", + "unknown": "Неизвестно" + }, + "fallbackPromoted": "Повышен до резерва", + "winnerCommitted": "Победитель подтверждён", + "roundStarted": "Раунд начат", + "events": { + "started": "Запрос запущен", + "ready": "Получены первые допустимые данные", + "held": "Выдача удержана правилами маршрутизации", + "fallbackPromoted": "Повышен до резерва", + "winnerCommitted": "Подтверждён как победитель", + "finished": "Завершено: {outcome}" + }, + "traceTruncated": "Часть событий опущена из-за ограничения хранилища.", + "summary": "Раундов: {rounds} · попыток: {attempts} · максимум активных: {maxActive}", + "config": "Снимок настроек", + "configConcurrency": "Параллельность", + "configMaxRounds": "Максимум раундов", + "configDiscoverySla": "SLA раунда", + "configStickySla": "SLA Sticky", + "configTotalTimeout": "Общий тайм-аут", + "configStickyCooldown": "Пауза после тайм-аута Sticky" + }, "logicTrace": { "title": "Цепочка решений", "noDecisionData": "Нет данных о решениях", diff --git a/messages/zh-CN/dashboard.json b/messages/zh-CN/dashboard.json index 9ccd3be60..94490ca05 100644 --- a/messages/zh-CN/dashboard.json +++ b/messages/zh-CN/dashboard.json @@ -379,6 +379,100 @@ "overridden": "已被供应商覆写", "tooltip": "客户端在请求体中声明的思考强度（output_config.effort），按原值显示，代理不会重命名或转换等级。" }, + "routingTrace": { + "title": "路由模式", + "modes": { + "discovery": "有界供应商 Discovery", + "legacy_hedge": "旧版 Hedge 竞速", + "legacy_serial": "旧版串行故障转移", + "single_upstream": "单上游模式" + }, + "bypassed": "本请求未使用 Discovery：{reason}", + "bypassReasons": { + "disabled": "Discovery 未启用", + "non_streaming": "非流式请求", + "retry_not_allowed": "该端点不允许重试", + "provider_switch_not_allowed": "该端点不允许切换供应商", + "raw_passthrough": "原始透传请求", + "unsupported_protocol": "Discovery 不支持该协议", + "websocket": "WebSocket 请求", + "streaming_hedge_disabled": "本请求已禁用流式竞速", + "raw_cross_provider_fallback": "已启用原始跨供应商故障转移", + "missing_session": "缺少 Session 标识", + "missing_key": "缺少 API Key 标识", + "rollout_ineligible": "本请求不在当前灰度范围内", + "redis_capability_unavailable": "Redis 绑定能力不可用", + "binding_conflict": "Session 绑定状态冲突", + "lease_conflict": "另一个请求持有 Discovery 租约", + "lease_unavailable": "Discovery 租约不可用", + "unknown": "未满足 Discovery 准入条件" + }, + "discoveryTitle": "Discovery 轮次", + "discoveryCompact": "{rounds} 轮 · {attempts} 次", + "roundsLabel": "轮次", + "attemptsLabel": "尝试", + "winnerOrigin": "赢家来源", + "viewDetails": "查看 Discovery 详情", + "stickyPhase": "Sticky 探测", + "round": "第 {round} 轮", + "attempts": "{count} 次尝试", + "noAttempts": "没有记录到供应商尝试", + "providerFallback": "供应商 {id}", + "elapsed": "+{elapsed}ms", + "priority": "优先级 {priority}", + "roles": { + "sticky": "Sticky", + "normal": "候选", + "fallback": "保底" + }, + "outcomes": { + "success": "成功", + "pending": "等待中", + "ready": "已就绪", + "held": "已就绪，暂缓交付", + "winner": "胜出", + "failed": "失败", + "cancelled": "已取消", + "timeout": "SLA 超时", + "client_abort": "客户端已断开", + "deadline": "总时限到期" + }, + "terminalOutcome": "请求结果", + "bindingResult": "Sticky 绑定", + "bindingActions": { + "create": "创建", + "renew": "续期", + "clear": "清除", + "none": "不变更" + }, + "bindingOutcomes": { + "updated": "已更新", + "cleared": "已清除", + "skipped": "已跳过", + "failed": "失败", + "unknown": "未知" + }, + "fallbackPromoted": "提升为保底请求", + "winnerCommitted": "已提交赢家", + "roundStarted": "轮次开始", + "events": { + "started": "请求已发出", + "ready": "已收到有效首字", + "held": "按路由规则暂缓交付", + "fallbackPromoted": "已提升为保底", + "winnerCommitted": "已提交为赢家", + "finished": "结束：{outcome}" + }, + "traceTruncated": "追踪事件达到存储上限，部分事件已省略。", + "summary": "{rounds} 轮 · {attempts} 次尝试 · 最大并发 {maxActive}", + "config": "配置快照", + "configConcurrency": "并发数", + "configMaxRounds": "最大轮数", + "configDiscoverySla": "每轮 SLA", + "configStickySla": "Sticky SLA", + "configTotalTimeout": "总超时", + "configStickyCooldown": "Sticky 超时冷却" + }, "logicTrace": { "title": "决策链", "noDecisionData": "暂无决策数据", diff --git a/messages/zh-TW/dashboard.json b/messages/zh-TW/dashboard.json index 51e10ed36..34505e7fc 100644 --- a/messages/zh-TW/dashboard.json +++ b/messages/zh-TW/dashboard.json @@ -379,6 +379,100 @@ "overridden": "已被供應商覆寫", "tooltip": "用戶端在請求體中宣告的思考強度（output_config.effort），按原值顯示，代理不會重新命名或轉換等級。" }, + "routingTrace": { + "title": "路由方式", + "modes": { + "discovery": "有界供應商 Discovery", + "legacy_hedge": "舊版 Hedge 競速", + "legacy_serial": "舊版串行故障轉移", + "single_upstream": "單上游模式" + }, + "bypassed": "本請求未使用 Discovery：{reason}", + "bypassReasons": { + "disabled": "Discovery 未啟用", + "non_streaming": "非串流請求", + "retry_not_allowed": "該端點不允許重試", + "provider_switch_not_allowed": "該端點不允許切換供應商", + "raw_passthrough": "原始透傳請求", + "unsupported_protocol": "Discovery 不支援該協議", + "websocket": "WebSocket 請求", + "streaming_hedge_disabled": "本請求已停用串流競速", + "raw_cross_provider_fallback": "已啟用原始跨供應商故障轉移", + "missing_session": "缺少 Session 識別", + "missing_key": "缺少 API Key 識別", + "rollout_ineligible": "本請求不在目前灰度範圍內", + "redis_capability_unavailable": "Redis 綁定能力不可用", + "binding_conflict": "Session 綁定狀態衝突", + "lease_conflict": "另一個請求持有 Discovery 租約", + "lease_unavailable": "Discovery 租約不可用", + "unknown": "未滿足 Discovery 准入條件" + }, + "discoveryTitle": "Discovery 輪次", + "discoveryCompact": "{rounds} 輪 · {attempts} 次", + "roundsLabel": "輪次", + "attemptsLabel": "嘗試", + "winnerOrigin": "贏家來源", + "viewDetails": "查看 Discovery 詳情", + "stickyPhase": "Sticky 探測", + "round": "第 {round} 輪", + "attempts": "{count} 次嘗試", + "noAttempts": "沒有記錄到供應商嘗試", + "providerFallback": "供應商 {id}", + "elapsed": "+{elapsed}ms", + "priority": "優先級 {priority}", + "roles": { + "sticky": "Sticky", + "normal": "候選", + "fallback": "備援" + }, + "outcomes": { + "success": "成功完成", + "pending": "處理中", + "ready": "已就緒", + "held": "已就緒，暫緩交付", + "winner": "勝出", + "failed": "失敗", + "cancelled": "已中止", + "timeout": "SLA 逾時", + "client_abort": "用戶端已斷開", + "deadline": "總時限到期" + }, + "terminalOutcome": "請求結果", + "bindingResult": "Sticky 綁定", + "bindingActions": { + "create": "建立", + "renew": "續期", + "clear": "解除綁定", + "none": "不變更" + }, + "bindingOutcomes": { + "updated": "綁定已更新", + "cleared": "綁定已解除", + "skipped": "已略過", + "failed": "失敗", + "unknown": "未知狀態" + }, + "fallbackPromoted": "提升為保底請求", + "winnerCommitted": "已提交贏家", + "roundStarted": "輪次開始", + "events": { + "started": "請求已送出", + "ready": "已收到有效首個字元", + "held": "依路由規則暫緩交付", + "fallbackPromoted": "已提升為備援", + "winnerCommitted": "已提交為贏家", + "finished": "結束：{outcome}" + }, + "traceTruncated": "追蹤事件達到儲存上限，部分事件已省略。", + "summary": "{rounds} 輪 · {attempts} 次嘗試 · 最大並發 {maxActive}", + "config": "設定快照", + "configConcurrency": "並發數", + "configMaxRounds": "最大輪數", + "configDiscoverySla": "每輪 SLA", + "configStickySla": "Sticky SLA", + "configTotalTimeout": "總逾時", + "configStickyCooldown": "Sticky 逾時冷卻" + }, "logicTrace": { "title": "決策鏈", "noDecisionData": "暫無決策資料", diff --git a/src/actions/usage-logs.ts b/src/actions/usage-logs.ts index e75f71fe3..026412b97 100644 --- a/src/actions/usage-logs.ts +++ b/src/actions/usage-logs.ts @@ -701,6 +701,7 @@ export async function getUsageLogsBatch( const snapshot = liveData.get(key); if (snapshot) { row._liveChain = snapshot; + row.routingTrace = snapshot.routingTrace ?? row.routingTrace; } } } diff --git a/src/app/[locale]/dashboard/logs/_components/error-details-dialog.test.tsx b/src/app/[locale]/dashboard/logs/_components/error-details-dialog.test.tsx index 40f753fe9..7267e7bd9 100644 --- a/src/app/[locale]/dashboard/logs/_components/error-details-dialog.test.tsx +++ b/src/app/[locale]/dashboard/logs/_components/error-details-dialog.test.tsx @@ -5,8 +5,10 @@ import { act } from "react"; import { NextIntlClientProvider } from "next-intl"; import { Window } from "happy-dom"; import { beforeEach, describe, expect, test, vi } from "vitest"; +import dashboardMessages from "../../../../../../messages/en/dashboard.json"; import ipDetailsMessages from "../../../../../../messages/en/ipDetails.json"; import providerChainMessages from "../../../../../../messages/en/provider-chain.json"; +import type { RoutingTraceV1 } from "@/types/routing-trace"; const hasSessionMessagesMock = vi.fn(); @@ -206,6 +208,8 @@ const messages = { endpoint: "Endpoint", }, details: { + routingTrace: dashboardMessages.logs.details.routingTrace, + modelAudit: dashboardMessages.logs.details.modelAudit, title: "Request Details", inProgress: "In progress", statusTitle: "Status: {status}", @@ -1169,6 +1173,251 @@ describe("error-details-dialog tabs", () => { }); }); +describe("error-details-dialog routing trace", () => { + const discoveryTrace: RoutingTraceV1 = { + version: 1, + mode: "discovery", + startedAt: 1_000, + updatedAt: 13_000, + discoveryEnabled: true, + eligible: true, + config: { + discoveryConcurrency: 2, + maxDiscoveryRounds: 2, + discoverySlaMs: 10_000, + stickySlaMs: 20_000, + racingTotalTimeoutMs: 60_000, + stickyTimeoutCooldownMs: 300_000, + }, + events: [ + { + type: "attempt_started", + at: 1_000, + elapsedMs: 0, + round: 0, + attemptId: "sticky:1", + attemptKind: "sticky", + provider: { id: 10, name: "sticky-provider", priority: 1 }, + }, + { + type: "attempt_finished", + at: 3_000, + elapsedMs: 2_000, + round: 0, + attemptId: "sticky:1", + attemptKind: "sticky", + provider: { id: 10, name: "sticky-provider", priority: 1 }, + outcome: "cancelled", + cancellationKind: "sticky_timeout", + }, + { + type: "round_started", + at: 3_000, + elapsedMs: 2_000, + round: 1, + }, + { + type: "attempt_started", + at: 3_010, + elapsedMs: 2_010, + round: 1, + attemptId: "normal:1", + attemptKind: "normal", + provider: { id: 11, name: "candidate-a", priority: 1 }, + }, + { + type: "attempt_held", + at: 3_100, + elapsedMs: 2_100, + round: 1, + attemptId: "normal:1", + attemptKind: "normal", + provider: { id: 11, name: "candidate-a", priority: 1 }, + }, + { + type: "attempt_started", + at: 3_020, + elapsedMs: 2_020, + round: 1, + attemptId: "normal:2", + attemptKind: "normal", + provider: { id: 12, name: "candidate-b", priority: 2 }, + }, + { + type: "winner_committed", + at: 4_000, + elapsedMs: 3_000, + round: 1, + attemptId: "normal:2", + attemptKind: "normal", + provider: { id: 12, name: "candidate-b", priority: 2 }, + statusCode: 200, + }, + ], + summary: { + outcome: "success", + statusCode: 200, + durationMs: 12_000, + ttfbMs: 3_000, + attemptsPerRequest: 3, + maxActiveAttempts: 2, + rounds: 1, + providerMs: 5_000, + fallbackPromotions: 0, + cancelFailures: 0, + winnerOrigin: "normal", + winnerProviderId: 12, + winnerRound: 1, + }, + }; + + test("renders Sticky and same-round attempts in the Discovery branch", () => { + const html = renderWithIntl( + + ); + const document = parseHtml(html); + + expect(document.querySelector("[data-testid='routing-mode-banner']")?.textContent).toContain( + "Bounded Discovery" + ); + expect(document.querySelector("[data-testid='discovery-round-0']")?.textContent).toContain( + "sticky-provider" + ); + const roundOne = document.querySelector("[data-testid='discovery-round-1']"); + expect(roundOne?.textContent).toContain("candidate-a"); + expect(roundOne?.textContent).toContain("candidate-b"); + expect(roundOne?.textContent).toContain("Ready, held"); + expect(roundOne?.textContent).toContain("Winner"); + expect(roundOne?.querySelector(".sm\\:grid-cols-2")).not.toBeNull(); + expect(html).toContain("60000ms"); + expect(html).toContain("300000ms"); + }); + + test("shows a legacy mode and bypass reason while retaining the old chain", () => { + const legacyTrace: RoutingTraceV1 = { + version: 1, + mode: "legacy_serial", + startedAt: 1_000, + updatedAt: 2_000, + discoveryEnabled: true, + eligible: false, + bypassReason: "non_streaming", + events: [], + }; + const html = renderWithIntl( + + ); + + expect(html).toContain("Legacy serial fallback"); + expect(html).toContain("Non-streaming request"); + expect(html).toContain("legacy-provider"); + expect(html).not.toContain("Discovery rounds"); + }); + + test("shows late terminal failure and Sticky binding result after a first-byte winner", () => { + const failedTrace: RoutingTraceV1 = { + ...discoveryTrace, + events: [ + ...discoveryTrace.events, + { + type: "request_finished", + at: 5_000, + elapsedMs: 4_000, + outcome: "failed", + statusCode: 502, + }, + { + type: "binding_finalized", + at: 5_100, + elapsedMs: 4_100, + provider: { id: 12, name: "candidate-b" }, + bindingAction: "renew", + outcome: "skipped", + reason: "generation_conflict", + }, + ], + summary: { ...discoveryTrace.summary!, outcome: "failed", statusCode: 502 }, + }; + const html = renderWithIntl( + + ); + const document = parseHtml(html); + const terminal = document.querySelector("[data-testid='discovery-terminal-status']"); + + expect(terminal?.textContent).toContain("Request result"); + expect(terminal?.textContent).toContain("Failed"); + expect(terminal?.textContent).toContain("HTTP 502"); + expect(terminal?.textContent).toContain("Sticky binding"); + expect(terminal?.textContent).toContain("Renew"); + expect(terminal?.textContent).toContain("generation_conflict"); + expect(document.querySelector("[data-testid='discovery-round-1']")?.textContent).toContain( + "Failed" + ); + }); + + test("derives live attempt concurrency while the terminal summary is not available", () => { + const { summary: _summary, ...liveTrace } = discoveryTrace; + const html = renderWithIntl( + + ); + const document = parseHtml(html); + const discoveryView = document.querySelector("[data-testid='discovery-trace-view']"); + + expect(discoveryView?.textContent).toContain("1 rounds · 3 attempts · max 2 active"); + expect(discoveryView?.textContent).not.toContain("0 attempts · max 0 active"); + }); + + test("falls back to the old chain for an unsupported trace version", () => { + const html = renderWithIntl( + + ); + + expect(html).toContain("old-provider"); + expect(html).not.toContain("Bounded Discovery"); + }); +}); + describe("error-details-dialog origin decision chain", () => { test("shows origin chain trigger for session reuse flow with sessionId", () => { const html = renderWithIntl( diff --git a/src/app/[locale]/dashboard/logs/_components/error-details-dialog/components/DiscoveryTraceView.tsx b/src/app/[locale]/dashboard/logs/_components/error-details-dialog/components/DiscoveryTraceView.tsx new file mode 100644 index 000000000..e5949ccd8 --- /dev/null +++ b/src/app/[locale]/dashboard/logs/_components/error-details-dialog/components/DiscoveryTraceView.tsx @@ -0,0 +1,649 @@ +"use client"; + +import { + CheckCircle, + ChevronRight, + CircleDot, + Clock3, + GitBranch, + Link2, + Server, + ShieldCheck, + XCircle, + Zap, +} from "lucide-react"; +import { useTranslations } from "next-intl"; +import { Badge } from "@/components/ui/badge"; +import { cn } from "@/lib/utils"; +import type { RoutingTraceV1 } from "@/types/routing-trace"; + +const KNOWN_BYPASS_REASONS = new Set([ + "disabled", + "non_streaming", + "retry_not_allowed", + "provider_switch_not_allowed", + "raw_passthrough", + "unsupported_protocol", + "websocket", + "streaming_hedge_disabled", + "raw_cross_provider_fallback", + "missing_session", + "missing_key", + "rollout_ineligible", + "redis_capability_unavailable", + "binding_conflict", + "lease_conflict", + "lease_unavailable", +]); + +type TraceRecord = Record; + +type AttemptView = { + id: string; + providerId: number | null; + providerName: string | null; + round: number; + role: "sticky" | "normal" | "fallback"; + priority: number | null; + startedAt: number | null; + elapsedMs: number | null; + outcome: + | "pending" + | "ready" + | "held" + | "winner" + | "failed" + | "cancelled" + | "timeout" + | "client_abort" + | "deadline"; + statusCode: number | null; + fallbackPromoted: boolean; + winnerCommitted: boolean; + history: Array<{ + type: string; + elapsedMs: number | null; + outcome: AttemptView["outcome"] | null; + }>; +}; + +function asRecord(value: unknown): TraceRecord { + return value && typeof value === "object" && !Array.isArray(value) ? (value as TraceRecord) : {}; +} + +function asString(value: unknown): string | null { + return typeof value === "string" && value.length > 0 ? value : null; +} + +function asNumber(value: unknown): number | null { + return typeof value === "number" && Number.isFinite(value) ? value : null; +} + +function eventType(event: TraceRecord): string { + return asString(event.type) ?? asString(event.event) ?? "unknown"; +} + +function eventProvider(event: TraceRecord): { + id: number | null; + name: string | null; + priority: number | null; +} { + const provider = asRecord(event.provider); + return { + id: asNumber(event.providerId) ?? asNumber(provider.id), + name: asString(event.providerName) ?? asString(provider.name), + priority: asNumber(event.priority) ?? asNumber(provider.priority), + }; +} + +function normalizeRole(value: unknown, round: number): AttemptView["role"] { + if (value === "fallback") return "fallback"; + if (value === "sticky" || round === 0) return "sticky"; + return "normal"; +} + +function normalizeOutcome(value: unknown): AttemptView["outcome"] | null { + switch (value) { + case "pending": + case "ready": + case "held": + case "winner": + case "failed": + case "cancelled": + case "timeout": + case "client_abort": + case "deadline": + return value; + case "sla_timeout": + return "timeout"; + default: + return null; + } +} + +function normalizeTerminalOutcome( + value: unknown +): "success" | "failed" | "client_abort" | "deadline" | "pending" { + switch (value) { + case "success": + case "failed": + case "client_abort": + case "deadline": + return value; + default: + return "pending"; + } +} + +function applyEventOutcome(attempt: AttemptView, type: string, event: TraceRecord): void { + const explicit = normalizeOutcome(event.outcome); + if (explicit) attempt.outcome = explicit; + + switch (type) { + case "attempt_ready": + attempt.outcome = "ready"; + break; + case "attempt_held": + attempt.outcome = "held"; + break; + case "attempt_failed": + attempt.outcome = "failed"; + break; + case "attempt_cancelled": + attempt.outcome = "cancelled"; + break; + case "fallback_promoted": + attempt.role = "fallback"; + attempt.fallbackPromoted = true; + break; + case "winner_committed": + attempt.outcome = "winner"; + attempt.winnerCommitted = true; + break; + } + + const cancellationKind = asString(event.cancellationKind); + if (cancellationKind === "client_abort") attempt.outcome = "client_abort"; + if (cancellationKind === "request_deadline") attempt.outcome = "deadline"; + if ( + cancellationKind === "discovery_sla_timeout" || + cancellationKind === "round_timeout" || + cancellationKind === "sticky_timeout" + ) { + attempt.outcome = "timeout"; + } +} + +function buildAttempts(trace: RoutingTraceV1): AttemptView[] { + const attempts = new Map(); + + for (const rawEvent of trace.events) { + const event = asRecord(rawEvent); + const type = eventType(event); + const provider = eventProvider(event); + const attemptId = + asString(event.attemptId) ?? + (provider.id != null && type.startsWith("attempt_") + ? `${provider.id}:${asNumber(event.sequence) ?? attempts.size + 1}` + : null); + if (!attemptId) continue; + + const round = Math.max(0, asNumber(event.round) ?? 0); + const existing = attempts.get(attemptId); + const attempt: AttemptView = existing ?? { + id: attemptId, + providerId: provider.id, + providerName: provider.name, + round, + role: normalizeRole(event.attemptKind ?? event.role ?? event.kind, round), + priority: provider.priority, + startedAt: null, + elapsedMs: null, + outcome: "pending", + statusCode: null, + fallbackPromoted: false, + winnerCommitted: false, + history: [], + }; + + attempt.providerId ??= provider.id; + attempt.providerName ??= provider.name; + attempt.round = Math.max(attempt.round, round); + attempt.priority ??= provider.priority; + attempt.statusCode ??= asNumber(event.statusCode); + const elapsedMs = asNumber(event.elapsedMs); + if (type === "attempt_started") attempt.startedAt = elapsedMs; + if (elapsedMs != null) attempt.elapsedMs = elapsedMs; + attempt.role = + attempt.role === "fallback" + ? "fallback" + : normalizeRole( + event.attemptKind ?? event.role ?? event.kind ?? attempt.role, + attempt.round + ); + applyEventOutcome(attempt, type, event); + if ( + type === "attempt_started" || + type === "attempt_ready" || + type === "attempt_held" || + type === "attempt_finished" || + type === "fallback_promoted" || + type === "winner_committed" + ) { + attempt.history.push({ + type, + elapsedMs, + outcome: normalizeOutcome(event.outcome), + }); + } + attempts.set(attemptId, attempt); + } + + const terminalEvent = trace.events.findLast((event) => event.type === "request_finished"); + const terminalOutcome = normalizeTerminalOutcome(terminalEvent?.outcome); + if (terminalEvent && terminalOutcome !== "success") { + for (const attempt of attempts.values()) { + if (!attempt.winnerCommitted) continue; + attempt.outcome = + terminalOutcome === "client_abort" || terminalOutcome === "deadline" + ? terminalOutcome + : "failed"; + attempt.statusCode = terminalEvent?.statusCode ?? attempt.statusCode; + } + } + + return [...attempts.values()].sort((a, b) => { + if (a.round !== b.round) return a.round - b.round; + return (a.startedAt ?? Number.MAX_SAFE_INTEGER) - (b.startedAt ?? Number.MAX_SAFE_INTEGER); + }); +} + +function numberFrom(record: TraceRecord, ...keys: string[]): number | null { + for (const key of keys) { + const value = asNumber(record[key]); + if (value != null) return value; + } + return null; +} + +function deriveRuntimeStats(trace: RoutingTraceV1): { + rounds: number; + attemptCount: number; + maxActive: number; +} { + const startedAttempts = new Set(); + const activeAttempts = new Set(); + let anonymousAttempts = 0; + let maxActive = 0; + let maxRound = 0; + + for (const event of trace.events) { + if (typeof event.round === "number" && Number.isFinite(event.round)) { + maxRound = Math.max(maxRound, event.round); + } + + if (event.type === "attempt_started") { + if (!event.attemptId) { + anonymousAttempts += 1; + continue; + } + if (!startedAttempts.has(event.attemptId)) { + startedAttempts.add(event.attemptId); + activeAttempts.add(event.attemptId); + maxActive = Math.max(maxActive, activeAttempts.size); + } + continue; + } + + if (event.type === "attempt_finished" && event.attemptId) { + activeAttempts.delete(event.attemptId); + } + } + + return { + rounds: maxRound, + attemptCount: startedAttempts.size + anonymousAttempts, + maxActive, + }; +} + +function outcomeStyle(outcome: AttemptView["outcome"]): { + icon: typeof Server; + className: string; +} { + switch (outcome) { + case "winner": + return { + icon: CheckCircle, + className: + "border-emerald-200 bg-emerald-50 text-emerald-700 dark:border-emerald-800 dark:bg-emerald-950/20 dark:text-emerald-300", + }; + case "failed": + return { + icon: XCircle, + className: + "border-rose-200 bg-rose-50 text-rose-700 dark:border-rose-800 dark:bg-rose-950/20 dark:text-rose-300", + }; + case "cancelled": + case "client_abort": + case "deadline": + return { + icon: XCircle, + className: + "border-slate-200 bg-slate-50 text-slate-600 dark:border-slate-700 dark:bg-slate-900/40 dark:text-slate-300", + }; + case "held": + case "timeout": + return { + icon: Clock3, + className: + "border-amber-200 bg-amber-50 text-amber-700 dark:border-amber-800 dark:bg-amber-950/20 dark:text-amber-300", + }; + case "ready": + return { + icon: ShieldCheck, + className: + "border-blue-200 bg-blue-50 text-blue-700 dark:border-blue-800 dark:bg-blue-950/20 dark:text-blue-300", + }; + default: + return { + icon: Server, + className: "border-border bg-muted/30 text-muted-foreground dark:bg-slate-900/30", + }; + } +} + +export function RoutingModeBanner({ trace }: { trace: RoutingTraceV1 }) { + const t = useTranslations("dashboard.logs.details.routingTrace"); + const Icon = + trace.mode === "discovery" ? Zap : trace.mode === "single_upstream" ? Server : GitBranch; + const rawReason = trace.bypassReason; + const reason = + rawReason && KNOWN_BYPASS_REASONS.has(rawReason) ? rawReason : rawReason ? "unknown" : null; + + return ( +
+
+ + {t("title")} + + {t(`modes.${trace.mode}`)} + +
+ {reason && ( +

+ {t("bypassed", { reason: t(`bypassReasons.${reason}`) })} +

+ )} +
+ ); +} + +export function DiscoveryTraceView({ trace }: { trace: RoutingTraceV1 }) { + const t = useTranslations("dashboard.logs.details.routingTrace"); + const attempts = buildAttempts(trace); + const grouped = new Map(); + for (const attempt of attempts) { + const group = grouped.get(attempt.round) ?? []; + group.push(attempt); + grouped.set(attempt.round, group); + } + + const summary = asRecord(trace.summary); + const config = asRecord(trace.config); + const runtimeStats = deriveRuntimeStats(trace); + const rounds = + numberFrom(summary, "rounds", "roundsVisited") ?? + Math.max(runtimeStats.rounds, ...attempts.map((attempt) => attempt.round)); + const attemptCount = + numberFrom(summary, "attemptsPerRequest", "attempts", "attemptsStarted") ?? + Math.max(runtimeStats.attemptCount, attempts.length); + const maxActive = numberFrom(summary, "maxActive", "maxActiveAttempts") ?? runtimeStats.maxActive; + const terminalEvent = trace.events.findLast((event) => event.type === "request_finished"); + const bindingEvent = trace.events.findLast((event) => event.type === "binding_finalized"); + const terminalOutcome = normalizeTerminalOutcome(terminalEvent?.outcome); + const bindingOutcome = + bindingEvent?.outcome === "updated" || + bindingEvent?.outcome === "cleared" || + bindingEvent?.outcome === "skipped" || + bindingEvent?.outcome === "failed" + ? bindingEvent.outcome + : "unknown"; + + return ( +
+
+
+

+ + {t("discoveryTitle")} +

+

+ {t("summary", { rounds, attempts: attemptCount, maxActive })} +

+
+ {trace.truncated && ( + + {t("traceTruncated")} + + )} +
+ + {(terminalEvent || bindingEvent) && ( +
+ {terminalEvent && ( +
+ {t("terminalOutcome")} +
+ {t(`outcomes.${terminalOutcome}`)} + {terminalEvent.statusCode != null && ( + HTTP {terminalEvent.statusCode} + )} +
+
+ )} + {bindingEvent && ( +
+ {t("bindingResult")} +
+
+ {t(`bindingActions.${bindingEvent.bindingAction ?? "none"}`)} ·{" "} + {t(`bindingOutcomes.${bindingOutcome}`)} +
+ {bindingEvent.reason && ( +
+ {bindingEvent.reason} +
+ )} +
+
+ )} +
+ )} + + {Object.keys(config).length > 0 && ( +
+
{t("config")}
+
+ {numberFrom(config, "discoveryConcurrency", "concurrency") != null && ( + + )} + {numberFrom(config, "maxDiscoveryRounds", "maxRounds") != null && ( + + )} + {numberFrom(config, "discoverySlaMs") != null && ( + + )} + {numberFrom(config, "stickySlaMs") != null && ( + + )} + {numberFrom(config, "racingTotalTimeoutMs", "totalTimeoutMs") != null && ( + + )} + {numberFrom(config, "stickyTimeoutCooldownMs") != null && ( + + )} +
+
+ )} + + {grouped.size === 0 ? ( +
+ + {t("noAttempts")} +
+ ) : ( +
+ {[...grouped.entries()].map(([round, roundAttempts]) => ( +
+
+
+ {round === 0 ? ( + + ) : ( + + )} +
+ {round === 0 ? t("stickyPhase") : t("round", { round })} +
+
+ + {t("attempts", { count: roundAttempts.length })} + +
+
+ {roundAttempts.map((attempt) => ( + + ))} +
+
+ ))} +
+ )} +
+ ); +} + +function TraceValue({ label, value }: { label: string; value: string | number | null }) { + return ( +
+
{label}
+
{value}
+
+ ); +} + +function AttemptCard({ attempt }: { attempt: AttemptView }) { + const t = useTranslations("dashboard.logs.details.routingTrace"); + const style = outcomeStyle(attempt.outcome); + const Icon = style.icon; + const providerName = + attempt.providerName ?? t("providerFallback", { id: attempt.providerId ?? "-" }); + + return ( +
+
+ +
+
+ + {providerName} + + {attempt.elapsedMs != null && ( + + {t("elapsed", { elapsed: Math.max(0, Math.round(attempt.elapsedMs)) })} + + )} +
+
+ + {t(`roles.${attempt.role}`)} + + + {t(`outcomes.${attempt.outcome}`)} + + {attempt.priority != null && ( + + {t("priority", { priority: attempt.priority })} + + )} + {attempt.statusCode != null && ( + HTTP {attempt.statusCode} + )} +
+ {(attempt.fallbackPromoted || attempt.winnerCommitted) && ( +
+ + {attempt.winnerCommitted ? t("winnerCommitted") : t("fallbackPromoted")} +
+ )} + {attempt.history.length > 0 && ( +
+ {attempt.history.map((event, index) => ( +
+ + + {formatAttemptEvent(t, event)} + + {event.elapsedMs != null && ( + + {t("elapsed", { elapsed: Math.max(0, Math.round(event.elapsedMs)) })} + + )} +
+ ))} +
+ )} +
+
+
+ ); +} + +function formatAttemptEvent( + t: ReturnType>, + event: AttemptView["history"][number] +): string { + switch (event.type) { + case "attempt_started": + return t("events.started"); + case "attempt_ready": + return t("events.ready"); + case "attempt_held": + return t("events.held"); + case "fallback_promoted": + return t("events.fallbackPromoted"); + case "winner_committed": + return t("events.winnerCommitted"); + case "attempt_finished": + return t("events.finished", { + outcome: t(`outcomes.${event.outcome ?? "pending"}`), + }); + default: + return event.type; + } +} diff --git a/src/app/[locale]/dashboard/logs/_components/error-details-dialog/components/LogicTraceTab.tsx b/src/app/[locale]/dashboard/logs/_components/error-details-dialog/components/LogicTraceTab.tsx index 90c1ece95..6985debb5 100644 --- a/src/app/[locale]/dashboard/logs/_components/error-details-dialog/components/LogicTraceTab.tsx +++ b/src/app/[locale]/dashboard/logs/_components/error-details-dialog/components/LogicTraceTab.tsx @@ -28,7 +28,9 @@ import { formatCurrency } from "@/lib/utils/currency"; import { findHedgeLoserCost, summarizeHedgeBilling } from "@/lib/utils/hedge-billing"; import { formatProbability, formatProviderTimeline } from "@/lib/utils/provider-chain-formatter"; import type { ProviderChainItem } from "@/types/message"; +import { normalizeRoutingTrace } from "@/types/routing-trace"; import { type LogicTraceTabProps, parseBlockedReason } from "../types"; +import { DiscoveryTraceView, RoutingModeBanner } from "./DiscoveryTraceView"; import { StepCard, type StepStatus } from "./StepCard"; function getRequestStatus(item: ProviderChainItem): StepStatus { @@ -63,6 +65,7 @@ function getRequestStatus(item: ProviderChainItem): StepStatus { export function LogicTraceTab({ statusCode: _statusCode, providerChain, + routingTrace, sessionId, blockedBy, blockedReason, @@ -184,10 +187,20 @@ export function LogicTraceTab({ // Count providers at each stage const totalProviders = decisionContext?.totalProviders || 0; const afterHealthCheck = decisionContext?.afterHealthCheck || 0; + const normalizedRoutingTrace = normalizeRoutingTrace(routingTrace); // Calculate step offset for session reuse flow const sessionReuseStepOffset = isSessionReuseFlow ? 1 : 0; + if (normalizedRoutingTrace?.mode === "discovery") { + return ( +
+ + +
+ ); + } + return (
{/* Warmup Skip Info */} @@ -243,6 +256,8 @@ export function LogicTraceTab({
)} + {normalizedRoutingTrace && } + {/* Decision Chain Header */} {providerChain && providerChain.length > 0 && (
diff --git a/src/app/[locale]/dashboard/logs/_components/error-details-dialog/components/index.ts b/src/app/[locale]/dashboard/logs/_components/error-details-dialog/components/index.ts index 641795a9a..67223ae82 100644 --- a/src/app/[locale]/dashboard/logs/_components/error-details-dialog/components/index.ts +++ b/src/app/[locale]/dashboard/logs/_components/error-details-dialog/components/index.ts @@ -1,3 +1,4 @@ +export { DiscoveryTraceView, RoutingModeBanner } from "./DiscoveryTraceView"; export { LatencyBreakdownBar } from "./LatencyBreakdownBar"; export { LogicTraceTab } from "./LogicTraceTab"; export { PerformanceTab } from "./PerformanceTab"; diff --git a/src/app/[locale]/dashboard/logs/_components/error-details-dialog/index.tsx b/src/app/[locale]/dashboard/logs/_components/error-details-dialog/index.tsx index 74a66d790..901dd1a75 100644 --- a/src/app/[locale]/dashboard/logs/_components/error-details-dialog/index.tsx +++ b/src/app/[locale]/dashboard/logs/_components/error-details-dialog/index.tsx @@ -11,6 +11,7 @@ import { hasSessionMessages } from "@/lib/api-client/v1/actions/active-sessions" import { cn } from "@/lib/utils"; import type { HedgeLoserBilling, StoredCostBreakdown } from "@/types/cost-breakdown"; import type { ProviderChainItem } from "@/types/message"; +import type { RoutingTraceV1 } from "@/types/routing-trace"; import type { SpecialSetting } from "@/types/special-settings"; import type { BillingModelSource } from "@/types/system-config"; import { LogicTraceTab, PerformanceTab, SummaryTab } from "./components"; @@ -19,6 +20,7 @@ interface ErrorDetailsDialogProps { statusCode: number | null; errorMessage: string | null; providerChain: ProviderChainItem[] | null; + routingTrace?: RoutingTraceV1 | null; sessionId: string | null; requestSequence?: number | null; blockedBy?: string | null; @@ -63,6 +65,7 @@ export function ErrorDetailsDialog({ statusCode, errorMessage, providerChain, + routingTrace, sessionId, requestSequence, blockedBy, @@ -212,6 +215,7 @@ export function ErrorDetailsDialog({ statusCode, errorMessage, providerChain, + routingTrace, sessionId, requestSequence, blockedBy, diff --git a/src/app/[locale]/dashboard/logs/_components/error-details-dialog/types.ts b/src/app/[locale]/dashboard/logs/_components/error-details-dialog/types.ts index ad65c070a..dc1ea1247 100644 --- a/src/app/[locale]/dashboard/logs/_components/error-details-dialog/types.ts +++ b/src/app/[locale]/dashboard/logs/_components/error-details-dialog/types.ts @@ -1,5 +1,6 @@ import type { HedgeLoserBilling, StoredCostBreakdown } from "@/types/cost-breakdown"; import type { ProviderChainItem } from "@/types/message"; +import type { RoutingTraceV1 } from "@/types/routing-trace"; import type { SpecialSetting } from "@/types/special-settings"; import type { BillingModelSource } from "@/types/system-config"; @@ -13,6 +14,8 @@ export interface TabSharedProps { errorMessage: string | null; /** Provider decision chain */ providerChain: ProviderChainItem[] | null; + /** Versioned routing trace for Discovery and legacy routing decisions */ + routingTrace?: RoutingTraceV1 | null; /** Session ID */ sessionId: string | null; /** Request sequence number within session */ diff --git a/src/app/[locale]/dashboard/logs/_components/provider-chain-popover.test.tsx b/src/app/[locale]/dashboard/logs/_components/provider-chain-popover.test.tsx index 4d65e49e2..7906ff832 100644 --- a/src/app/[locale]/dashboard/logs/_components/provider-chain-popover.test.tsx +++ b/src/app/[locale]/dashboard/logs/_components/provider-chain-popover.test.tsx @@ -3,7 +3,9 @@ import { renderToStaticMarkup } from "react-dom/server"; import { NextIntlClientProvider } from "next-intl"; import { Window } from "happy-dom"; import { describe, expect, test, vi } from "vitest"; +import dashboardMessages from "../../../../../../messages/en/dashboard.json"; import providerChainMessages from "../../../../../../messages/en/provider-chain.json"; +import type { RoutingTraceV1 } from "@/types/routing-trace"; vi.mock("@/lib/utils/provider-chain-formatter", async (importOriginal) => { const actual = await importOriginal(); @@ -85,6 +87,7 @@ const messages = { decisionChain: "Decision chain", }, details: { + routingTrace: dashboardMessages.logs.details.routingTrace, clickStatusCode: "Click status code", fake200ForwardedNotice: "Note: payload may have been forwarded", fake200DetectedReason: "Detected reason: {reason}", @@ -480,3 +483,179 @@ describe("provider-chain-popover hedge/abort reason handling", () => { expect(html).toContain("p1"); }); }); + +describe("provider-chain-popover Discovery summary", () => { + test("shows rounds, attempts and winner origin instead of the legacy serial chain", () => { + const routingTrace: RoutingTraceV1 = { + version: 1, + mode: "discovery", + startedAt: 1_000, + updatedAt: 3_000, + discoveryEnabled: true, + eligible: true, + events: [], + summary: { + outcome: "success", + statusCode: 200, + durationMs: 2_000, + ttfbMs: 1_500, + attemptsPerRequest: 4, + maxActiveAttempts: 2, + rounds: 2, + providerMs: 3_400, + fallbackPromotions: 1, + cancelFailures: 0, + winnerOrigin: "fallback", + winnerProviderId: 2, + winnerRound: 1, + }, + }; + + const html = renderWithIntl( + undefined} + /> + ); + + expect(html).toContain("2R · 4 tries"); + expect(html).toContain("Winner origin"); + expect(html).toContain("Fallback"); + expect(html).toContain("View Discovery details"); + expect(html).not.toContain("1 times"); + }); + + test("derives live rounds and attempt count before a terminal summary exists", () => { + const routingTrace: RoutingTraceV1 = { + version: 1, + mode: "discovery", + startedAt: 1_000, + updatedAt: 3_000, + discoveryEnabled: true, + eligible: true, + events: [ + { type: "round_started", at: 1_000, elapsedMs: 0, round: 1 }, + { + type: "attempt_started", + at: 1_010, + elapsedMs: 10, + round: 1, + attemptId: "normal:1", + attemptKind: "normal", + provider: { id: 1, name: "candidate-a" }, + }, + { + type: "attempt_started", + at: 1_020, + elapsedMs: 20, + round: 1, + attemptId: "normal:2", + attemptKind: "normal", + provider: { id: 2, name: "candidate-b" }, + }, + { + type: "attempt_finished", + at: 2_000, + elapsedMs: 1_000, + round: 1, + attemptId: "normal:1", + attemptKind: "normal", + outcome: "cancelled", + }, + { type: "round_started", at: 2_010, elapsedMs: 1_010, round: 2 }, + { + type: "attempt_started", + at: 2_020, + elapsedMs: 1_020, + round: 2, + attemptId: "normal:3", + attemptKind: "normal", + provider: { id: 3, name: "candidate-c" }, + }, + ], + }; + + const html = renderWithIntl( + + ); + + expect(html).toContain("2R · 3 tries"); + expect(html).toContain("Request result"); + expect(html).toContain("Pending"); + expect(html).not.toContain("0R · 0 tries"); + }); + + test("uses the terminal failure over a committed winner and preserves fake-200 warnings", () => { + const routingTrace: RoutingTraceV1 = { + version: 1, + mode: "discovery", + startedAt: 1_000, + updatedAt: 5_000, + discoveryEnabled: true, + eligible: true, + events: [ + { + type: "winner_committed", + at: 2_000, + elapsedMs: 1_000, + round: 1, + attemptId: "normal:1", + attemptKind: "normal", + provider: { id: 1, name: "candidate-a" }, + statusCode: 200, + }, + { + type: "request_finished", + at: 5_000, + elapsedMs: 4_000, + outcome: "failed", + statusCode: 502, + reason: "FAKE_200_EMPTY_BODY", + }, + ], + summary: { + outcome: "success", + statusCode: 200, + durationMs: 1_000, + ttfbMs: 1_000, + attemptsPerRequest: 1, + maxActiveAttempts: 1, + rounds: 1, + providerMs: 1_000, + fallbackPromotions: 0, + cancelFailures: 0, + winnerOrigin: "normal", + winnerProviderId: 1, + winnerRound: 1, + }, + }; + + const html = renderWithIntl( + + ); + const document = parseHtml(html); + const terminal = document.querySelector("[data-testid='discovery-compact-terminal']"); + + expect(terminal?.textContent).toContain("Failed"); + expect(terminal?.textContent).toContain("HTTP 502"); + expect(terminal?.innerHTML).toContain("text-rose-600"); + expect(terminal?.innerHTML).not.toContain("text-emerald-600"); + expect(html).toContain("Detected reason: Empty response body"); + expect(html).toContain("Note: payload may have been forwarded"); + expect(html).toContain("Why CCH cannot retry this response on the server"); + }); +}); diff --git a/src/app/[locale]/dashboard/logs/_components/provider-chain-popover.tsx b/src/app/[locale]/dashboard/logs/_components/provider-chain-popover.tsx index 958257769..af08d4a4e 100644 --- a/src/app/[locale]/dashboard/logs/_components/provider-chain-popover.tsx +++ b/src/app/[locale]/dashboard/logs/_components/provider-chain-popover.tsx @@ -4,6 +4,7 @@ import { AlertTriangle, CheckCircle, ChevronRight, + Clock3, GitBranch, InfoIcon, Link2, @@ -26,11 +27,13 @@ import { } from "@/lib/utils/provider-chain-formatter"; import { parseProviderGroups } from "@/lib/utils/provider-group"; import type { ProviderChainItem } from "@/types/message"; +import { normalizeRoutingTrace, type RoutingTraceV1 } from "@/types/routing-trace"; import { getFake200ReasonKey } from "./fake200-reason"; import { Fake200RetryTooltip } from "./fake200-retry-tooltip"; interface ProviderChainPopoverProps { chain: ProviderChainItem[]; + routingTrace?: RoutingTraceV1 | null; finalProvider: string; /** Whether a cost badge is displayed, affects name max width */ hasCostBadge?: boolean; @@ -42,6 +45,51 @@ function parseGroupTags(groupTag?: string | null): string[] { return Array.from(new Set(parseProviderGroups(groupTag))); } +function deriveDiscoveryStats(trace: RoutingTraceV1): { + rounds: number; + attempts: number; +} { + const startedAttempts = new Set(); + let anonymousAttempts = 0; + let maxRound = 0; + + for (const event of trace.events) { + if (typeof event.round === "number" && Number.isFinite(event.round)) { + maxRound = Math.max(maxRound, event.round); + } + if (event.type !== "attempt_started") continue; + if (event.attemptId) startedAttempts.add(event.attemptId); + else anonymousAttempts += 1; + } + + return { + rounds: trace.summary?.rounds ?? maxRound, + attempts: trace.summary?.attemptsPerRequest ?? startedAttempts.size + anonymousAttempts, + }; +} + +type DiscoveryTerminalOutcome = "success" | "failed" | "client_abort" | "deadline" | "pending"; + +function getDiscoveryTerminal(trace: RoutingTraceV1): { + outcome: DiscoveryTerminalOutcome; + statusCode: number | null; +} { + const terminalEvent = trace.events.findLast((event) => event.type === "request_finished"); + const rawOutcome = terminalEvent?.outcome ?? trace.summary?.outcome; + const outcome: DiscoveryTerminalOutcome = + rawOutcome === "success" || + rawOutcome === "failed" || + rawOutcome === "client_abort" || + rawOutcome === "deadline" + ? rawOutcome + : "pending"; + + return { + outcome, + statusCode: terminalEvent?.statusCode ?? trace.summary?.statusCode ?? null, + }; +} + /** * Get status icon and color for a provider chain item */ @@ -126,12 +174,15 @@ function getItemStatus(item: ProviderChainItem): { export function ProviderChainPopover({ chain, + routingTrace, finalProvider, hasCostBadge = false, onChainItemClick, }: ProviderChainPopoverProps) { const t = useTranslations("dashboard"); const tChain = useTranslations("provider-chain"); + const tRouting = useTranslations("dashboard.logs.details.routingTrace"); + const normalizedRoutingTrace = normalizeRoutingTrace(routingTrace); // “假 200”识别发生在 SSE 流式结束后：此时响应内容可能已透传给客户端，但内部会按失败统计/熔断。 const hasFake200PostStreamFailure = chain.some( @@ -151,6 +202,133 @@ export function ProviderChainPopover({ // Fallback for empty string const displayName = finalProvider || "-"; + if (normalizedRoutingTrace?.mode === "discovery") { + const { rounds, attempts } = deriveDiscoveryStats(normalizedRoutingTrace); + const terminal = getDiscoveryTerminal(normalizedRoutingTrace); + const terminalPresentation = + terminal.outcome === "success" + ? { icon: CheckCircle, className: "text-emerald-600" } + : terminal.outcome === "failed" + ? { icon: XCircle, className: "text-rose-600" } + : terminal.outcome === "deadline" + ? { icon: Clock3, className: "text-amber-600" } + : terminal.outcome === "client_abort" + ? { icon: MinusCircle, className: "text-amber-600" } + : { icon: RefreshCw, className: "text-muted-foreground" }; + const TerminalIcon = terminalPresentation.icon; + const rawWinnerOrigin = normalizedRoutingTrace.summary?.winnerOrigin; + const winnerOrigin = + rawWinnerOrigin === "sticky" || rawWinnerOrigin === "normal" || rawWinnerOrigin === "fallback" + ? rawWinnerOrigin + : "none"; + + return ( + + + + + +
+

+ + {tRouting("discoveryTitle")} +

+ + {tRouting("modes.discovery")} + +
+
+
+
+
{tRouting("roundsLabel")}
+
{rounds}
+
+
+
{tRouting("attemptsLabel")}
+
{attempts}
+
+
+
+ + {tRouting("terminalOutcome")}: + + {tRouting(`outcomes.${terminal.outcome}`)} + + {terminal.statusCode != null && ( + HTTP {terminal.statusCode} + )} +
+ {winnerOrigin !== "none" && ( +
+ + {tRouting("winnerOrigin")}: + + {tRouting(`roles.${winnerOrigin}`)} + +
+ )} +
+ {(hasFake200PostStreamFailure || onChainItemClick) && ( +
+ {hasFake200PostStreamFailure && ( +
+
+ )} + {onChainItemClick && ( + + )} +
+ )} + + + ); + } + // Determine max width based on whether cost badge is present const maxWidthClass = hasCostBadge ? "max-w-[140px]" : "max-w-[180px]"; diff --git a/src/app/[locale]/dashboard/logs/_components/usage-logs-table.tsx b/src/app/[locale]/dashboard/logs/_components/usage-logs-table.tsx index 017325e0f..b8511dc2a 100644 --- a/src/app/[locale]/dashboard/logs/_components/usage-logs-table.tsx +++ b/src/app/[locale]/dashboard/logs/_components/usage-logs-table.tsx @@ -217,6 +217,7 @@ export function UsageLogsTable({
undefined); return await discoveryPromise; } - if (ProxyForwarder.shouldUseStreamingHedge(session)) { + const useStreamingHedge = ProxyForwarder.shouldUseStreamingHedge(session); + const singleUpstream = + discoveryPreparation.reason === "binding_conflict" || + discoveryPreparation.reason === "lease_conflict" || + (session.isStreamingHedgeDisabled() && !session.isSessionBindingAllowed()) || + !ProxyForwarder.getEndpointPolicy(session).allowRetry || + !ProxyForwarder.getEndpointPolicy(session).allowProviderSwitch; + session.initializeRoutingTrace({ + mode: singleUpstream + ? "single_upstream" + : useStreamingHedge + ? "legacy_hedge" + : "legacy_serial", + discoveryEnabled: discoverySettings.discoveryEnabled === true, + eligible: false, + bypassReason: discoveryPreparation.reason, + startedAt: requestStartedAt, + }); + + if (useStreamingHedge) { const hedgePromise = ProxyForwarder.sendStreamingWithHedge(session); void hedgePromise.catch(() => undefined); return await hedgePromise; @@ -3931,38 +3992,38 @@ export class ProxyForwarder { session: ProxySession, settings: SystemSettings, requestStartedAt: number - ): Promise { + ): Promise { if (settings.discoveryEnabled !== true) { - return null; + return { status: "skipped", reason: "disabled" }; } const endpointPolicy = ProxyForwarder.getEndpointPolicy(session); const protocol = ProxyForwarder.discoveryProtocol(session); const message = session.request.message as Record; - if ( - !endpointPolicy.allowRetry || - !endpointPolicy.allowProviderSwitch || - message?.stream !== true || - endpointPolicy.bypassForwarderPreprocessing || - protocol === "unknown" || - isWebsocketClientRequest(session.headers) || - session.isStreamingHedgeDisabled() || - session.isRawCrossProviderFallbackEnabled() - ) { - return null; - } + if (!endpointPolicy.allowRetry) return { status: "skipped", reason: "retry_not_allowed" }; + if (!endpointPolicy.allowProviderSwitch) + return { status: "skipped", reason: "provider_switch_not_allowed" }; + if (message?.stream !== true) return { status: "skipped", reason: "non_streaming" }; + if (endpointPolicy.bypassForwarderPreprocessing) + return { status: "skipped", reason: "raw_passthrough" }; + if (protocol === "unknown") return { status: "skipped", reason: "unsupported_protocol" }; + if (isWebsocketClientRequest(session.headers)) + return { status: "skipped", reason: "websocket" }; + if (session.isStreamingHedgeDisabled()) + return { status: "skipped", reason: "streaming_hedge_disabled" }; + if (session.isRawCrossProviderFallbackEnabled()) + return { status: "skipped", reason: "raw_cross_provider_fallback" }; const sessionId = session.sessionId; const keyId = session.authState?.key?.id ?? session.messageContext?.key?.id ?? null; - if (!sessionId || keyId == null) { - return null; - } + if (!sessionId) return { status: "skipped", reason: "missing_session" }; + if (keyId == null) return { status: "skipped", reason: "missing_key" }; if (!isDiscoveryRolloutEligible(keyId, sessionId, getEnvConfig().DISCOVERY_ROLLOUT_PERCENT)) { - return null; + return { status: "skipped", reason: "rollout_ineligible" }; } const capabilityState = await SessionManager.ensureVersionedBindingCapability(); if (capabilityState !== "available") { - return null; + return { status: "skipped", reason: "redis_capability_unavailable" }; } let bindingSnapshot = session.getSessionBindingSnapshot(); @@ -3981,7 +4042,11 @@ export class ProxyForwarder { session.disableStreamingHedge(); session.setSessionBindingAllowed(false); } - return null; + return { + status: "skipped", + reason: + binding.status === "conflict" ? "binding_conflict" : "redis_capability_unavailable", + }; } bindingSnapshot = binding.snapshot; session.setSessionBindingSnapshot(binding.snapshot); @@ -4010,18 +4075,24 @@ export class ProxyForwarder { keyId, }); } - return null; + return { + status: "skipped", + reason: lease.status === "conflict" ? "lease_conflict" : "lease_unavailable", + }; } return { - settings, - bindingSnapshot, - requestStartedAt, - lease: { - sessionId, - keyId, - ownerToken: lease.ownerToken, - ttlSeconds, + status: "prepared", + prepared: { + settings, + bindingSnapshot, + requestStartedAt, + lease: { + sessionId, + keyId, + ownerToken: lease.ownerToken, + ttlSeconds, + }, }, }; } @@ -5069,6 +5140,8 @@ export class ProxyForwarder { pending: boolean; ready: boolean; round: number; + traceRound: number; + traceFinished: boolean; readerTransferred: boolean; readerCancelled: boolean; providerSessionRefOwned: boolean; @@ -5103,12 +5176,67 @@ export class ProxyForwarder { let stickyTimeoutWaveClaim: StickyTimeoutWaveReservation | null = null; let stickyTimeoutWaveLaunchPromise: Promise | null = null; let stickyTimeoutCooldownPromise: Promise | null = null; + const tracedFallbackPromotions = new Set(); + const tracedRounds = new Set(); const hasSticky = session.shouldReuseProvider() && !!session.sessionId && bindingSnapshot.providerId === initialProvider.id; let stickyProbeActive = hasSticky; - if (hasSticky) coordinator.startStickyProbe(); + if (hasSticky) { + coordinator.startStickyProbe(); + session.appendRoutingTraceEvent({ + type: "sticky_probe_started", + round: 0, + attemptKind: "sticky", + provider: { + id: initialProvider.id, + name: initialProvider.name, + priority: ProxyProviderResolver.resolveEffectivePriorityForSession( + initialProvider, + session + ), + }, + }); + } + + const recordFallbackPromotion = ( + attemptId: string, + providerId: number, + round: number, + provider?: Provider + ) => { + discoveryMetrics.fallbackPromoted(attemptId, providerId, round); + if (tracedFallbackPromotions.has(attemptId)) return; + tracedFallbackPromotions.add(attemptId); + session.appendRoutingTraceEvent({ + type: "fallback_promoted", + attemptId, + attemptKind: "fallback", + round, + provider: { + id: providerId, + ...(provider?.name ? { name: provider.name } : {}), + ...(provider + ? { + priority: ProxyProviderResolver.resolveEffectivePriorityForSession( + provider, + session + ), + } + : {}), + }, + }); + }; + const recordRoundStarted = (round: number, slots: number) => { + if (tracedRounds.has(round)) return; + tracedRounds.add(round); + session.appendRoutingTraceEvent({ + type: "round_started", + round, + reason: `slots:${slots}`, + }); + }; const waitForRoundLaunches = (): Promise => { if (roundLaunchesInProgress === 0) return Promise.resolve(); return new Promise((resolve) => roundLaunchIdleWaiters.add(resolve)); @@ -5234,7 +5362,8 @@ export class ProxyForwarder { const cleanupAttempt = ( attempt: (typeof winner & { id: string }) | null, - cancellationKind: DiscoveryCancellationKind | null + cancellationKind: DiscoveryCancellationKind | null, + failure?: { statusCode?: number; reason?: string } ) => { if (attempt?.readerTransferred) return; if (!attempt) return; @@ -5282,6 +5411,28 @@ export class ProxyForwarder { outcome: cancellationKind ? "cancelled" : "failed", cancellationKind, }); + if (!attempt.traceFinished) { + attempt.traceFinished = true; + session.appendRoutingTraceEvent({ + type: "attempt_finished", + attemptId: attempt.id, + attemptKind: + attempt.traceRound === 0 && attempt.kind === "normal" ? "sticky" : attempt.kind, + round: attempt.traceRound, + provider: { + id: attempt.provider.id, + name: attempt.provider.name, + priority: ProxyProviderResolver.resolveEffectivePriorityForSession( + attempt.provider, + session + ), + }, + outcome: cancellationKind ? "cancelled" : "failed", + ...(cancellationKind ? { cancellationKind } : {}), + ...(failure?.statusCode != null ? { statusCode: failure.statusCode } : {}), + ...(failure?.reason ? { reason: failure.reason } : {}), + }); + } }; const cancelAttempt = ( @@ -5376,16 +5527,18 @@ export class ProxyForwarder { } } const statusCode = error instanceof ProxyError ? error.statusCode : 503; - discoveryMetrics.finish({ - outcome: - options.cancellationKind === "client_abort" || statusCode === 499 - ? "client_abort" - : options.cancellationKind === "request_deadline" - ? "deadline" - : "failed", - statusCode, - winnerOrigin: "none", - }); + session.setRoutingTraceSummary( + discoveryMetrics.snapshot({ + outcome: + options.cancellationKind === "client_abort" || statusCode === 499 + ? "client_abort" + : options.cancellationKind === "request_deadline" + ? "deadline" + : "failed", + statusCode, + winnerOrigin: "none", + }) + ); resolveResult?.({ error }); }; @@ -5414,16 +5567,52 @@ export class ProxyForwarder { providerId: attempt.provider.id, outcome: "winner", }); - discoveryMetrics.finish({ - outcome: "success", + const winnerOrigin = + attempt.traceRound === 0 && attempt.kind === "normal" ? "sticky" : attempt.kind; + if (!attempt.traceFinished) { + attempt.traceFinished = true; + session.appendRoutingTraceEvent({ + type: "attempt_finished", + attemptId: attempt.id, + attemptKind: winnerOrigin, + round: attempt.traceRound, + provider: { + id: attempt.provider.id, + name: attempt.provider.name, + priority: ProxyProviderResolver.resolveEffectivePriorityForSession( + attempt.provider, + session + ), + }, + outcome: "winner", + statusCode: attempt.response.status, + }); + } + session.appendRoutingTraceEvent({ + type: "winner_committed", + attemptId: attempt.id, + attemptKind: winnerOrigin, + round: attempt.traceRound, + provider: { + id: attempt.provider.id, + name: attempt.provider.name, + priority: ProxyProviderResolver.resolveEffectivePriorityForSession( + attempt.provider, + session + ), + }, + outcome: "winner", statusCode: attempt.response.status, - winnerOrigin: attempt.kind, - winnerProviderId: attempt.provider.id, - winnerRound: - hasSticky && attempt.provider.id === initialProvider.id && stickyProbeActive - ? 0 - : attempt.round, }); + session.setRoutingTraceSummary( + discoveryMetrics.snapshot({ + outcome: "success", + statusCode: attempt.response.status, + winnerOrigin, + winnerProviderId: attempt.provider.id, + winnerRound: attempt.traceRound, + }) + ); session.setProvider(attempt.provider); if (attempt.session !== session) ProxyForwarder.syncWinningAttemptSession(session, attempt.session); @@ -5729,6 +5918,13 @@ export class ProxyForwarder { } const controller = new AbortController(); const id = `${provider.id}:${sequence + 1}`; + const isStickyAttempt = + stickyProbeActive && provider.id === initialProvider.id && effectiveKind === "normal"; + const traceRound = isStickyAttempt ? 0 : currentRound; + const effectivePriority = ProxyProviderResolver.resolveEffectivePriorityForSession( + provider, + session + ); const attempt = { id, kind: effectiveKind, @@ -5738,6 +5934,8 @@ export class ProxyForwarder { pending: true, ready: false, round: currentRound, + traceRound, + traceFinished: false, readerTransferred: false, readerCancelled: false, providerSessionRefOwned: providerSessionRefTracked, @@ -5780,6 +5978,8 @@ export class ProxyForwarder { pending: boolean; ready: boolean; round: number; + traceRound: number; + traceFinished: boolean; readerTransferred: boolean; readerCancelled: boolean; providerSessionRefOwned: boolean; @@ -5790,7 +5990,7 @@ export class ProxyForwarder { const registered = coordinator.addAttempt({ id, providerId: provider.id, - priority: ProxyProviderResolver.resolveEffectivePriorityForSession(provider, session), + priority: effectivePriority, kind: effectiveKind, ready: false, pending: true, @@ -5812,12 +6012,20 @@ export class ProxyForwarder { discoveryMetrics.attemptStarted({ attemptId: id, providerId: provider.id, - round: - stickyProbeActive && provider.id === initialProvider.id && effectiveKind === "normal" - ? 0 - : currentRound, + round: traceRound, kind: effectiveKind, }); + session.appendRoutingTraceEvent({ + type: "attempt_started", + attemptId: id, + attemptKind: isStickyAttempt ? "sticky" : effectiveKind, + round: traceRound, + provider: { + id: provider.id, + name: provider.name, + priority: effectivePriority, + }, + }); void ProxyForwarder.doForward( attempt.session, @@ -5871,6 +6079,22 @@ export class ProxyForwarder { throw new ProxyError("Invalid upstream discovery response", 502); if (!validity.ready) continue; attempt.ready = true; + session.appendRoutingTraceEvent({ + type: "attempt_ready", + attemptId: id, + attemptKind: + attempt.traceRound === 0 && attempt.kind === "normal" ? "sticky" : attempt.kind, + round: attempt.traceRound, + provider: { + id: provider.id, + name: provider.name, + priority: ProxyProviderResolver.resolveEffectivePriorityForSession( + provider, + session + ), + }, + outcome: "ready", + }); if ( attempt.kind === "fallback" && (fallbackPromotionBlocked || @@ -5882,6 +6106,15 @@ export class ProxyForwarder { // without promoting so the total deadline can still recover the // buffered fallback if that setup stalls. coordinator.recordReadyHeld(id); + session.appendRoutingTraceEvent({ + type: "attempt_held", + attemptId: id, + attemptKind: "fallback", + round: attempt.traceRound, + provider: { id: provider.id, name: provider.name }, + outcome: "held", + reason: "replacement_wave_pending", + }); return; } // Record readiness even when the priority gate holds this attempt. @@ -5896,7 +6129,19 @@ export class ProxyForwarder { // The coordinator owns priority gating. A ready lower-priority // candidate stays held while a higher tier is still pending. Stop // reading so later chunks are not consumed before promotion. - if (action.type === "none") return; + if (action.type === "none") { + session.appendRoutingTraceEvent({ + type: "attempt_held", + attemptId: id, + attemptKind: + attempt.traceRound === 0 && attempt.kind === "normal" ? "sticky" : attempt.kind, + round: attempt.traceRound, + provider: { id: provider.id, name: provider.name }, + outcome: "held", + reason: "priority_gate", + }); + return; + } return; } }) @@ -5964,7 +6209,10 @@ export class ProxyForwarder { request: buildRequestDetails(session), }, }); - cleanupAttempt(attempt, null); + cleanupAttempt(attempt, null, { + statusCode: lastError instanceof ProxyError ? lastError.statusCode : 503, + reason: "local_overload", + }); await settleFailure(lastError, { preserveBinding: true }); return; } @@ -6013,7 +6261,10 @@ export class ProxyForwarder { attempt.providerSessionRefOwned = false; } attempt.pending = false; - cleanupAttempt(attempt, null); + cleanupAttempt(attempt, null, { + statusCode: lastError instanceof ProxyError ? lastError.statusCode : undefined, + reason: "rectifier_retry", + }); session.addProviderToChain(provider, { ...buildRetryFailedChainEntry( provider, @@ -6130,7 +6381,13 @@ export class ProxyForwarder { ) { await recordFailure(provider.id, lastError).catch(() => undefined); } - cleanupAttempt(attempt, null); + cleanupAttempt(attempt, null, { + statusCode: lastError instanceof ProxyError ? lastError.statusCode : undefined, + reason: + lastErrorCategory == null + ? "unknown_error" + : ErrorCategory[lastErrorCategory].toLowerCase(), + }); if (lastErrorCategory === ErrorCategory.NON_RETRYABLE_CLIENT_ERROR) { // Client/input errors are independent of the selected provider. // Stop Discovery immediately so the same invalid request is not @@ -6439,6 +6696,7 @@ export class ProxyForwarder { // launch batch is still in flight. if (!slotState.timerStarted && !committed && !settled) { clearRoundTimer(); + recordRoundStarted(currentRound, requestedSlots()); scheduleRoundBoundary(discoverySlaMs); slotState.timerStarted = true; } @@ -6629,13 +6887,18 @@ export class ProxyForwarder { const retrySetupReservation = retrySetupReservations.get(action.promoteAttemptId); if (fallback) { fallback.kind = "fallback"; - discoveryMetrics.fallbackPromoted(fallback.id, fallback.provider.id, fallback.round); + recordFallbackPromotion( + fallback.id, + fallback.provider.id, + fallback.traceRound, + fallback.provider + ); } if (retrySetupReservation) { const epoch = coordinator.epochs; retrySetupReservation.requestEpoch = epoch.requestEpoch; retrySetupReservation.roundEpoch = epoch.roundEpoch; - discoveryMetrics.fallbackPromoted( + recordFallbackPromotion( action.promoteAttemptId, retrySetupReservation.providerId, coordinator.round @@ -6686,6 +6949,7 @@ export class ProxyForwarder { const orchestrate = async () => { let initialLaunchFailed = false; + if (!hasSticky) recordRoundStarted(currentRound, concurrency); try { await launch(initialProvider, "normal"); } catch (error) { @@ -6722,6 +6986,20 @@ export class ProxyForwarder { ? (attempts.get(stickyRetryReservation.placeholderAttemptId) ?? null) : null); if (stickyAttempt || stickyRetryReservation) { + session.appendRoutingTraceEvent({ + type: "sticky_timeout", + round: 0, + attemptKind: "sticky", + provider: { + id: initialProvider.id, + name: initialProvider.name, + priority: ProxyProviderResolver.resolveEffectivePriorityForSession( + initialProvider, + session + ), + }, + outcome: "timeout", + }); const stickyAttemptId = stickyRetryReservation?.placeholderAttemptId ?? stickyAttempt?.id; if (!stickyAttemptId || !coordinator.demoteToFallback(stickyAttemptId)) return; @@ -6733,7 +7011,7 @@ export class ProxyForwarder { stickyRetryReservation.requestEpoch = epoch.requestEpoch; stickyRetryReservation.roundEpoch = epoch.roundEpoch; } - discoveryMetrics.fallbackPromoted(stickyAttemptId, initialProvider.id, 0); + recordFallbackPromotion(stickyAttemptId, initialProvider.id, 0, initialProvider); fallbackPromotionBlocked = true; stickyTimeoutWaveReservation = { fallbackAttemptId: stickyAttemptId, diff --git a/src/app/v1/_lib/proxy/response-handler.ts b/src/app/v1/_lib/proxy/response-handler.ts index 64bbfa2b9..9d6f96f88 100644 --- a/src/app/v1/_lib/proxy/response-handler.ts +++ b/src/app/v1/_lib/proxy/response-handler.ts @@ -13,7 +13,6 @@ import { recordDiscoveryControlEvent } from "@/lib/observability/discovery-metri import { requestCloudPriceTableSync } from "@/lib/price-sync/cloud-price-updater"; import { ProxyStatusTracker } from "@/lib/proxy-status-tracker"; import { RateLimitService } from "@/lib/rate-limit"; -import { deleteLiveChain } from "@/lib/redis/live-chain-store"; import type { SessionBindingSnapshot } from "@/lib/redis/session-binding"; import { SessionManager } from "@/lib/session-manager"; import { SessionTracker } from "@/lib/session-tracker"; @@ -42,6 +41,7 @@ import { updateMessageRequestCostWithBreakdown, updateMessageRequestDetailsDurably, updateMessageRequestDetailsIfUnfinalized, + updateMessageRequestRoutingTrace, updateMessageRequestWinnerCost, } from "@/repository/message"; import type { HedgeLoserBilling, StoredCostBreakdown } from "@/types/cost-breakdown"; @@ -1496,6 +1496,33 @@ function finalizeDeferredStreamingFinalizationIfNeeded( const meta = consumeDeferredStreamingFinalization(session); const provider = session.provider; const providerIdForPersistence = meta?.providerId ?? provider?.id ?? null; + let bindingTraceRecorded = false; + const recordBindingFinalized = async (options: { + bindingAction: "create" | "renew" | "clear" | "none"; + outcome: string; + reason?: string; + }): Promise => { + if (bindingTraceRecorded || !meta?.discoveryLease) return; + bindingTraceRecorded = true; + session.appendRoutingTraceEvent({ + type: "binding_finalized", + provider: + providerIdForPersistence == null + ? undefined + : { + id: providerIdForPersistence, + ...(meta.providerName ? { name: meta.providerName } : {}), + priority: meta.providerPriority, + }, + bindingAction: options.bindingAction, + outcome: options.outcome, + ...(options.reason ? { reason: options.reason } : {}), + }); + const routingTrace = session.getRoutingTrace(); + if (routingTrace && session.messageContext) { + await updateMessageRequestRoutingTrace(session.messageContext.id, routingTrace); + } + }; const clearSessionBinding = async () => { if (!session.sessionId || !isSessionBindingMutationAllowed(session)) return; const hedgeAuthority = meta?.isHedgeWinner @@ -1520,7 +1547,14 @@ function finalizeDeferredStreamingFinalizationIfNeeded( return; } const keyId = session.authState?.key?.id ?? session.messageContext?.key?.id ?? null; - if (meta?.bindingIntent === "none" || meta?.bindingIntent === "create") return; + if (meta?.bindingIntent === "none" || meta?.bindingIntent === "create") { + await recordBindingFinalized({ + bindingAction: "none", + outcome: "skipped", + reason: meta.bindingIntent === "create" ? "stream_not_successful" : "binding_not_requested", + }); + return; + } if (meta?.bindingIntent === "renew") { // A client disconnect is not evidence that the Sticky Provider failed. // Discovery renewals may only clear the exact binding snapshot that was @@ -1539,6 +1573,11 @@ function finalizeDeferredStreamingFinalizationIfNeeded( expectedProviderId: meta.providerId, reason: "missing_or_mismatched_snapshot", }); + await recordBindingFinalized({ + bindingAction: "clear", + outcome: "skipped", + reason: "missing_or_mismatched_snapshot", + }); return; } if (!(await discoveryLeaseLifecycle.ensureOwned())) { @@ -1550,6 +1589,11 @@ function finalizeDeferredStreamingFinalizationIfNeeded( expectedProviderId: meta.bindingSnapshot.providerId, } ); + await recordBindingFinalized({ + bindingAction: "clear", + outcome: "skipped", + reason: "discovery_lease_not_owned", + }); return; } const cleared = await SessionManager.clearVersionedSessionProvider( @@ -1565,6 +1609,11 @@ function finalizeDeferredStreamingFinalizationIfNeeded( reason: cleared.reason, }); } + await recordBindingFinalized({ + bindingAction: "clear", + outcome: cleared.status === "ok" ? "cleared" : "skipped", + reason: cleared.status === "ok" ? "stream_failed" : cleared.reason, + }); return; } @@ -1597,7 +1646,23 @@ function finalizeDeferredStreamingFinalizationIfNeeded( }; } - const cas = await SessionManager.compareAndSetSessionProvider(snapshot, providerId); + let cas: Awaited>; + try { + cas = await SessionManager.compareAndSetSessionProvider(snapshot, providerId); + } catch (error) { + logger.warn("[ResponseHandler] Discovery binding CAS failed", { + sessionId: snapshot.sessionId, + keyId, + providerId, + error: error instanceof Error ? error.message : String(error), + }); + await recordBindingFinalized({ + bindingAction: meta?.bindingIntent === "create" ? "create" : "renew", + outcome: "failed", + reason: "binding_error", + }); + return { updated: false, reason: "binding_error", details: "binding_error" }; + } if (cas.status === "conflict") { recordDiscoveryControlEvent("binding_cas_conflict", { requestId: session.messageContext?.id ?? null, @@ -1608,6 +1673,12 @@ function finalizeDeferredStreamingFinalizationIfNeeded( }); } + await recordBindingFinalized({ + bindingAction: meta?.bindingIntent === "create" ? "create" : "renew", + outcome: cas.status === "ok" ? "updated" : "skipped", + reason: cas.status === "ok" ? "generation_cas" : cas.reason, + }); + return { updated: cas.status === "ok", reason: cas.status === "ok" ? "discovery_generation_cas" : cas.reason, @@ -1636,6 +1707,11 @@ function finalizeDeferredStreamingFinalizationIfNeeded( }; const finalizeFailedDiscoveryBinding = async () => { settlePrimaryDiscoveryBinding(false); + await recordBindingFinalized({ + bindingAction: "none", + outcome: "skipped", + reason: "stream_not_successful", + }); await finalizeProviderSessionRef(); }; const confirmAuxiliarySessionBinding = async () => @@ -2045,8 +2121,8 @@ function finalizeDeferredStreamingFinalizationIfNeeded( const hedgeBindingCompletion = meta.hedgeBindingHeartbeat?.complete(); const commitSideEffects = async () => { let primaryDiscoveryBindingUpdated = false; - await hedgeBindingCompletion; try { + await hedgeBindingCompletion; if (meta.endpointId != null) { try { const { recordEndpointSuccess } = await import("@/lib/endpoint-circuit-breaker"); @@ -2095,6 +2171,14 @@ function finalizeDeferredStreamingFinalizationIfNeeded( keyId ); + if (isDiscoveryBinding && !bindingTraceRecorded) { + await recordBindingFinalized({ + bindingAction: meta.bindingIntent === "create" ? "create" : "renew", + outcome: "skipped", + reason: result.reason, + }); + } + primaryDiscoveryBindingUpdated = isDiscoveryBinding && result.updated; retainProviderSessionRef = primaryDiscoveryBindingUpdated && meta.providerSessionRefRetainOnSuccess === true; @@ -2134,6 +2218,22 @@ function finalizeDeferredStreamingFinalizationIfNeeded( } } + if (meta.discoveryLease && !bindingTraceRecorded) { + await recordBindingFinalized({ + bindingAction: + meta.bindingIntent === "create" || meta.bindingIntent === "renew" + ? meta.bindingIntent + : "none", + outcome: "skipped", + reason: + meta.bindingIntent === "none" + ? "fallback_winner" + : clientAborted + ? "client_aborted" + : "binding_not_permitted", + }); + } + logger.info("[ResponseHandler] Streaming request finalized as success", { providerId: meta.providerId, providerName: meta.providerName, @@ -2467,6 +2567,7 @@ export class ProxyResponseHandler { ...errorDetails, ttfbMs: session.ttfbMs ?? duration, providerChain: session.getProviderChain(), + routingTrace: session.finalizeRoutingTrace(finalizedStatusCode), model: session.getCurrentModel() ?? undefined, providerId: session.provider?.id, context1mApplied: session.getContext1mApplied(), @@ -2631,6 +2732,7 @@ export class ProxyResponseHandler { ...errorDetails, ttfbMs: session.ttfbMs ?? duration, providerChain: session.getProviderChain(), + routingTrace: session.finalizeRoutingTrace(finalizedStatusCode), model: session.getCurrentModel() ?? undefined, // 更新重定向后的模型 providerId: session.provider?.id, // 更新最终供应商ID（重试切换后） context1mApplied: session.getContext1mApplied(), @@ -2945,6 +3047,7 @@ export class ProxyResponseHandler { cacheCreation1hInputTokens: usageMetrics?.cache_creation_1h_input_tokens, cacheTtlApplied: usageMetrics?.cache_ttl ?? null, providerChain: session.getProviderChain(), + routingTrace: session.finalizeRoutingTrace(statusCode), ...(terminalErrorMessage ? { errorMessage: terminalErrorMessage } : {}), model: session.getCurrentModel() ?? undefined, // 更新重定向后的模型 actualResponseModel: extractActualResponseModelForProvider( @@ -4273,6 +4376,7 @@ export class ProxyResponseHandler { cacheCreation1hInputTokens: usageForCost?.cache_creation_1h_input_tokens, cacheTtlApplied: usageForCost?.cache_ttl ?? null, providerChain: session.getProviderChain(), + routingTrace: session.finalizeRoutingTrace(effectiveStatusCode), ...(streamErrorMessage ? { errorMessage: streamErrorMessage } : {}), model: currentRequestedModel ?? undefined, // 更新重定向后的模型 actualResponseModel: finalActualResponseModel, @@ -5804,6 +5908,7 @@ export async function finalizeRequestStats( ...(errorMessage ? { errorMessage } : {}), ttfbMs: session.ttfbMs ?? duration, providerChain: session.getProviderChain(), + routingTrace: session.finalizeRoutingTrace(statusCode), model: session.getCurrentModel() ?? undefined, actualResponseModel: extractActualResponseModelForProvider( provider.providerType, @@ -5921,6 +6026,7 @@ export async function finalizeRequestStats( cacheCreation1hInputTokens: normalizedUsage.cache_creation_1h_input_tokens, cacheTtlApplied: normalizedUsage.cache_ttl ?? null, providerChain: session.getProviderChain(), + routingTrace: session.finalizeRoutingTrace(statusCode), ...(errorMessage ? { errorMessage } : {}), model: session.getCurrentModel() ?? undefined, actualResponseModel: extractActualResponseModelForProvider( @@ -5941,7 +6047,7 @@ export async function finalizeRequestStats( if (session.sessionId && session.requestSequence != null) { if (session.shouldTrackSessionObservability()) { - void deleteLiveChain(session.sessionId, session.requestSequence); + void session.closeLiveObservability(); } } @@ -6167,6 +6273,7 @@ async function persistRequestFailure(options: { errorCause, ttfbMs: phase === "non-stream" ? (session.ttfbMs ?? duration) : session.ttfbMs, providerChain: session.getProviderChain(), + routingTrace: session.finalizeRoutingTrace(statusCode), model: session.getCurrentModel() ?? undefined, providerId: session.provider?.id, // 更新最终供应商ID（重试切换后） context1mApplied: session.getContext1mApplied(), @@ -6180,7 +6287,7 @@ async function persistRequestFailure(options: { if (session.sessionId && session.requestSequence != null) { if (session.shouldTrackSessionObservability()) { - void deleteLiveChain(session.sessionId, session.requestSequence); + void session.closeLiveObservability(); } } diff --git a/src/app/v1/_lib/proxy/session.ts b/src/app/v1/_lib/proxy/session.ts index 0ef7901c1..d0472c940 100644 --- a/src/app/v1/_lib/proxy/session.ts +++ b/src/app/v1/_lib/proxy/session.ts @@ -1,6 +1,10 @@ import type { Context } from "hono"; import { logger } from "@/lib/logger"; -import { writeLiveChain } from "@/lib/redis/live-chain-store"; +import { + deleteLiveChain, + writeLiveChain, + writeLiveRoutingTrace, +} from "@/lib/redis/live-chain-store"; import type { SessionBindingSnapshot } from "@/lib/redis/session-binding"; import { clientRequestsContext1m as clientRequestsContext1mHelper } from "@/lib/special-attributes"; import { @@ -14,6 +18,16 @@ import type { Key } from "@/types/key"; import type { ProviderChainItem } from "@/types/message"; import type { ModelPriceData } from "@/types/model-price"; import type { Provider, ProviderType } from "@/types/provider"; +import { + ROUTING_TRACE_MAX_EVENTS, + ROUTING_TRACE_VERSION, + type RoutingTraceConfigV1, + type RoutingTraceEventV1, + type RoutingTraceMode, + type RoutingTraceRequestOutcome, + type RoutingTraceSummaryV1, + type RoutingTraceV1, +} from "@/types/routing-trace"; import type { SpecialSetting } from "@/types/special-settings"; import type { BillingModelSource, CodexPriorityBillingSource } from "@/types/system-config"; import type { User } from "@/types/user"; @@ -154,6 +168,17 @@ export class ProxySession { // 上游决策链（记录尝试的供应商列表） private providerChain: ProviderChainItem[]; + // Request-level routing observability. Discovery attempts live here rather + // than providerChain because providerChain is also a billing/retry contract. + private routingTrace: RoutingTraceV1 | null = null; + private routingTraceSummaryDraft: RoutingTraceSummaryV1 | null = null; + private liveChainDirty = false; + private liveRoutingTraceDirty = false; + private liveObservabilityFlushPromise: Promise | null = null; + private liveObservabilityClosePromise: Promise | null = null; + private liveObservabilityClosed = false; + private routingTraceTerminalLogged = false; + // 上次选择的决策上下文（用于记录到 providerChain） private _lastSelectionContext?: ProviderChainItem["decisionContext"]; @@ -729,7 +754,234 @@ export class ProxySession { private persistLiveChain(): void { if (!this.sessionId || this.requestSequence == null) return; if (!this.shouldTrackSessionObservability()) return; - void writeLiveChain(this.sessionId, this.requestSequence, this.providerChain); + if (this.liveObservabilityClosed) return; + this.liveChainDirty = true; + this.scheduleLiveObservabilityFlush(); + } + + private scheduleLiveObservabilityFlush(): void { + if (this.liveObservabilityClosed || this.liveObservabilityFlushPromise) return; + const flush = Promise.resolve().then(() => this.flushLiveObservability()); + this.liveObservabilityFlushPromise = flush.finally(() => { + this.liveObservabilityFlushPromise = null; + if (!this.liveObservabilityClosed && (this.liveChainDirty || this.liveRoutingTraceDirty)) { + this.scheduleLiveObservabilityFlush(); + } + }); + } + + private async flushLiveObservability(): Promise { + while (this.liveChainDirty || this.liveRoutingTraceDirty) { + const writeChain = this.liveChainDirty; + const writeRoutingTrace = this.liveRoutingTraceDirty && this.routingTrace !== null; + this.liveChainDirty = false; + this.liveRoutingTraceDirty = false; + + const chain = writeChain ? structuredClone(this.providerChain) : null; + const routingTrace = writeRoutingTrace ? structuredClone(this.routingTrace) : null; + const writes: Promise[] = []; + if (chain) { + writes.push( + writeLiveChain(this.sessionId as string, this.requestSequence as number, chain) + ); + } + if (routingTrace) { + writes.push( + writeLiveRoutingTrace( + this.sessionId as string, + this.requestSequence as number, + routingTrace + ) + ); + } + + const results = await Promise.allSettled(writes); + for (const result of results) { + if (result.status === "rejected") { + logger.debug("[ProxySession] Failed to persist live routing observability", { + error: result.reason, + }); + } + } + } + } + + private persistLiveRoutingTrace(): void { + if (!this.sessionId || this.requestSequence == null || !this.routingTrace) return; + if (!this.shouldTrackSessionObservability()) return; + if (this.liveObservabilityClosed) return; + this.liveRoutingTraceDirty = true; + this.scheduleLiveObservabilityFlush(); + } + + initializeRoutingTrace(options: { + mode: RoutingTraceMode; + discoveryEnabled: boolean; + eligible: boolean; + bypassReason?: string; + config?: RoutingTraceConfigV1; + startedAt?: number; + }): void { + const now = Date.now(); + this.routingTrace = { + version: ROUTING_TRACE_VERSION, + mode: options.mode, + startedAt: options.startedAt ?? this.startTime, + updatedAt: now, + discoveryEnabled: options.discoveryEnabled, + eligible: options.eligible, + ...(options.bypassReason ? { bypassReason: options.bypassReason } : {}), + ...(options.config ? { config: structuredClone(options.config) } : {}), + events: [ + { + type: "request_started", + at: now, + elapsedMs: Math.max(0, now - (options.startedAt ?? this.startTime)), + reason: options.bypassReason, + }, + ], + }; + this.persistLiveRoutingTrace(); + } + + appendRoutingTraceEvent( + event: Omit & + Partial> + ): void { + if (!this.routingTrace) return; + const at = event.at ?? Date.now(); + const normalized: RoutingTraceEventV1 = { + ...event, + at, + elapsedMs: event.elapsedMs ?? Math.max(0, at - this.routingTrace.startedAt), + }; + let changed = false; + if (this.routingTrace.events.length < ROUTING_TRACE_MAX_EVENTS) { + this.routingTrace.events.push(normalized); + changed = true; + } else { + if (this.routingTrace.truncated !== true) { + this.routingTrace.truncated = true; + changed = true; + } + if ( + normalized.type === "winner_committed" || + normalized.type === "binding_finalized" || + normalized.type === "request_finished" + ) { + const replaceIndex = this.routingTrace.events.findIndex( + (existing) => + existing.type !== "winner_committed" && + existing.type !== "binding_finalized" && + existing.type !== "request_finished" + ); + if (replaceIndex >= 0) this.routingTrace.events.splice(replaceIndex, 1); + else this.routingTrace.events.shift(); + this.routingTrace.events.push(normalized); + changed = true; + } + } + if (!changed) return; + this.routingTrace.updatedAt = at; + this.persistLiveRoutingTrace(); + } + + setRoutingTraceSummary(summary: RoutingTraceSummaryV1): void { + if (!this.routingTrace) return; + // A first-byte winner is not a terminal success. Keep aggregate counters + // request-local until ResponseHandler completes stream validation. + this.routingTraceSummaryDraft = structuredClone(summary); + } + + finalizeRoutingTrace( + statusCode: number, + outcome?: RoutingTraceRequestOutcome + ): RoutingTraceV1 | null { + if (!this.routingTrace) return null; + const resolvedOutcome = + outcome ?? + (statusCode === 499 + ? "client_abort" + : this.routingTraceSummaryDraft?.outcome === "deadline" || + this.routingTrace.summary?.outcome === "deadline" + ? "deadline" + : statusCode >= 200 && statusCode < 400 + ? "success" + : "failed"); + const now = Date.now(); + const summaryBase = this.routingTraceSummaryDraft ?? this.routingTrace.summary; + if (summaryBase) { + this.routingTrace.summary = { + ...summaryBase, + outcome: resolvedOutcome, + statusCode, + durationMs: Math.max(0, now - this.routingTrace.startedAt), + ttfbMs: this.ttfbMs, + }; + } + const terminalEvent = this.routingTrace.events.find( + (event) => event.type === "request_finished" + ); + if (!terminalEvent) { + this.appendRoutingTraceEvent({ + type: "request_finished", + outcome: resolvedOutcome, + statusCode, + }); + } else { + terminalEvent.at = now; + terminalEvent.elapsedMs = Math.max(0, now - this.routingTrace.startedAt); + terminalEvent.outcome = resolvedOutcome; + terminalEvent.statusCode = statusCode; + this.routingTrace.updatedAt = now; + this.persistLiveRoutingTrace(); + } + return this.getRoutingTrace(); + } + + getRoutingTrace(): RoutingTraceV1 | null { + return this.routingTrace ? structuredClone(this.routingTrace) : null; + } + + private logRoutingTraceTerminalSummary(): void { + const summary = this.routingTrace?.summary; + if (this.routingTraceTerminalLogged || this.routingTrace?.mode !== "discovery" || !summary) { + return; + } + this.routingTraceTerminalLogged = true; + logger.info("[DiscoveryMetric] Request aggregate", { + event: "request_finished", + requestId: this.messageContext?.id ?? null, + sessionId: this.sessionId, + keyId: this.authState?.key?.id ?? this.messageContext?.key?.id ?? null, + outcome: summary.outcome, + statusCode: summary.statusCode, + winnerOrigin: summary.winnerOrigin, + winnerProviderId: summary.winnerProviderId, + winnerRound: summary.winnerRound, + elapsedMs: summary.durationMs, + ttfbMs: summary.ttfbMs, + attemptsPerRequest: summary.attemptsPerRequest, + maxActiveAttempts: summary.maxActiveAttempts, + rounds: summary.rounds, + providerMs: summary.providerMs, + fallbackPromotions: summary.fallbackPromotions, + cancelFailures: summary.cancelFailures, + }); + } + + async closeLiveObservability(): Promise { + if (this.liveObservabilityClosePromise) return this.liveObservabilityClosePromise; + this.scheduleLiveObservabilityFlush(); + this.liveObservabilityClosed = true; + this.liveObservabilityClosePromise = (async () => { + await (this.liveObservabilityFlushPromise ?? Promise.resolve()); + this.logRoutingTraceTerminalSummary(); + if (!this.sessionId || this.requestSequence == null) return; + if (!this.shouldTrackSessionObservability()) return; + await deleteLiveChain(this.sessionId, this.requestSequence); + })(); + return this.liveObservabilityClosePromise; } /** diff --git a/src/drizzle/schema.ts b/src/drizzle/schema.ts index 7534e86d2..539095ed1 100644 --- a/src/drizzle/schema.ts +++ b/src/drizzle/schema.ts @@ -21,6 +21,7 @@ import type { AllowedModelRuleInput, ProviderModelRedirectRule, ProviderType } f import type { FilterOperation } from "@/lib/request-filter-types"; import type { IpExtractionConfig } from "@/types/ip-extraction"; import type { AuditCategory } from "@/types/audit-log"; +import type { RoutingTraceV1 } from "@/types/routing-trace"; // Enums export const dailyResetModeEnum = pgEnum('daily_reset_mode', ['fixed', 'rolling']); @@ -487,6 +488,9 @@ export const messageRequest = pgTable('message_request', { // 上游决策链（记录尝试的供应商列表） providerChain: jsonb('provider_chain').$type>(), + // 请求路由轨迹（Discovery/legacy 模式、轮次、并发尝试与终态摘要） + routingTrace: jsonb('routing_trace').$type(), + // HTTP 状态码 statusCode: integer('status_code'), diff --git a/src/lib/ledger-backfill/trigger.sql b/src/lib/ledger-backfill/trigger.sql index 4a261b1ce..06e1e2bef 100644 --- a/src/lib/ledger-backfill/trigger.sql +++ b/src/lib/ledger-backfill/trigger.sql @@ -257,6 +257,36 @@ $$ LANGUAGE plpgsql; DROP TRIGGER IF EXISTS trg_upsert_usage_ledger ON message_request; CREATE TRIGGER trg_upsert_usage_ledger -AFTER INSERT OR UPDATE ON message_request +AFTER INSERT OR UPDATE OF + blocked_by, + status_code, + error_message, + provider_chain, + actual_response_model, + endpoint, + provider_id, + user_id, + "key", + model, + original_model, + api_type, + session_id, + cost_usd, + cost_multiplier, + group_cost_multiplier, + input_tokens, + output_tokens, + cache_creation_input_tokens, + cache_read_input_tokens, + cache_creation_5m_input_tokens, + cache_creation_1h_input_tokens, + cache_ttl_applied, + context_1m_applied, + swap_cache_ttl_applied, + duration_ms, + ttfb_ms, + client_ip, + created_at +ON message_request FOR EACH ROW EXECUTE FUNCTION fn_upsert_usage_ledger(); diff --git a/src/lib/observability/discovery-metrics.ts b/src/lib/observability/discovery-metrics.ts index 16cd93135..0e0706907 100644 --- a/src/lib/observability/discovery-metrics.ts +++ b/src/lib/observability/discovery-metrics.ts @@ -1,4 +1,5 @@ import { logger } from "@/lib/logger"; +import type { RoutingTraceSummaryV1 } from "@/types/routing-trace"; export type DiscoveryLifecycleEvent = | "request_started" @@ -11,7 +12,7 @@ export type DiscoveryLifecycleEvent = | "binding_cas_conflict" | "request_finished"; -export type DiscoveryWinnerOrigin = "normal" | "fallback" | "none"; +export type DiscoveryWinnerOrigin = "sticky" | "normal" | "fallback" | "none"; type DiscoveryMetricIdentity = { requestId: number | string | null; @@ -36,7 +37,7 @@ export class DiscoveryRequestMetrics { private providerMs = 0; private fallbackPromotions = 0; private cancelFailures = 0; - private finished = false; + private summary: RoutingTraceSummaryV1 | null = null; constructor( private readonly identity: DiscoveryMetricIdentity, @@ -99,18 +100,32 @@ export class DiscoveryRequestMetrics { }); } - finish(context: { + snapshot(context: { outcome: "success" | "failed" | "client_abort" | "deadline"; statusCode: number; winnerOrigin?: DiscoveryWinnerOrigin; winnerProviderId?: number | null; winnerRound?: number | null; - }): void { - if (this.finished) return; - this.finished = true; + }): RoutingTraceSummaryV1 { + if (this.summary) return this.summary; const elapsedMs = Math.max(0, Date.now() - this.startedAt); - logger.info("[DiscoveryMetric] Request aggregate", { - event: "request_finished", + this.summary = { + outcome: context.outcome, + statusCode: context.statusCode, + durationMs: elapsedMs, + ttfbMs: context.outcome === "success" ? elapsedMs : null, + attemptsPerRequest: this.attempts, + maxActiveAttempts: this.maxActive, + rounds: this.maxRound, + providerMs: this.providerMs, + fallbackPromotions: this.fallbackPromotions, + cancelFailures: this.cancelFailures, + winnerOrigin: context.winnerOrigin ?? "none", + winnerProviderId: context.winnerProviderId ?? null, + winnerRound: context.winnerRound ?? null, + }; + logger.debug("[DiscoveryMetric] Aggregate snapshot", { + event: context.outcome === "success" ? "winner_committed" : "request_failed", ...this.identity, ...context, elapsedMs, @@ -122,5 +137,6 @@ export class DiscoveryRequestMetrics { fallbackPromotions: this.fallbackPromotions, cancelFailures: this.cancelFailures, }); + return this.summary; } } diff --git a/src/lib/redis/live-chain-store.storage.test.ts b/src/lib/redis/live-chain-store.storage.test.ts new file mode 100644 index 000000000..d2a9492ef --- /dev/null +++ b/src/lib/redis/live-chain-store.storage.test.ts @@ -0,0 +1,219 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { LiveChainSnapshot } from "./live-chain-store"; +import type { RoutingTraceV1 } from "@/types/routing-trace"; + +interface FakeStore { + data: Map; + deleted: string[]; +} + +const stores = vi.hoisted(() => new Map()); + +vi.mock("./redis-kv-store", () => ({ + RedisKVStore: class { + private readonly backend: FakeStore; + + constructor({ prefix }: { prefix: string }) { + const existing = stores.get(prefix); + this.backend = existing ?? { + data: new Map(), + deleted: [], + }; + stores.set(prefix, this.backend); + } + + async set(key: string, value: T): Promise { + this.backend.data.set(key, value); + return true; + } + + async get(key: string): Promise { + return (this.backend.data.get(key) as T | undefined) ?? null; + } + + async delete(key: string): Promise { + this.backend.deleted.push(key); + return this.backend.data.delete(key); + } + }, +})); + +import { + deleteLiveChain, + readLiveChain, + readLiveChainBatch, + writeLiveChain, + writeLiveRoutingTrace, +} from "./live-chain-store"; + +const CHAIN_PREFIX = "cch:live-chain:"; +const TRACE_PREFIX = "cch:live-routing-trace:"; + +function makeTrace(overrides: Partial = {}): RoutingTraceV1 { + return { + version: 1, + mode: "discovery", + startedAt: 100, + updatedAt: 200, + discoveryEnabled: true, + eligible: true, + events: [ + { + type: "round_started", + at: 100, + elapsedMs: 0, + round: 1, + }, + ], + ...overrides, + }; +} + +function getStore(prefix: string): FakeStore { + const store = stores.get(prefix); + if (!store) throw new Error(`Missing fake store for ${prefix}`); + return store; +} + +describe("live-chain routing trace storage", () => { + beforeEach(() => { + for (const store of stores.values()) { + store.data.clear(); + store.deleted.length = 0; + } + }); + + it("stores routing trace separately and derives the live Discovery phase", async () => { + const trace = makeTrace(); + await writeLiveChain("session-a", 1, [ + { + id: 1, + name: "provider-a", + reason: "initial_selection", + timestamp: 100, + }, + ]); + await writeLiveRoutingTrace("session-a", 1, trace); + + expect(getStore(CHAIN_PREFIX).data.get("session-a:1")).not.toHaveProperty("routingTrace"); + await expect(readLiveChain("session-a", 1)).resolves.toMatchObject({ + phase: "discovery_racing", + routingTrace: trace, + }); + }); + + it("returns an early trace before the provider chain snapshot exists", async () => { + const trace = makeTrace({ + updatedAt: 150, + events: [{ type: "sticky_probe_started", at: 150, elapsedMs: 0 }], + }); + await writeLiveRoutingTrace("early", 1, trace); + + await expect(readLiveChain("early", 1)).resolves.toEqual({ + chain: [], + phase: "discovery_sticky", + updatedAt: 150, + routingTrace: trace, + }); + }); + + it("does not erase a concurrent trace when the legacy chain is updated", async () => { + const trace = makeTrace({ + events: [{ type: "fallback_promoted", at: 200, elapsedMs: 100, round: 1 }], + }); + await writeLiveRoutingTrace("session-a", 2, trace); + await writeLiveChain("session-a", 2, [ + { id: 2, name: "provider-b", reason: "retry_failed", timestamp: 200 }, + ]); + + await expect(readLiveChain("session-a", 2)).resolves.toMatchObject({ + phase: "discovery_fallback", + routingTrace: trace, + }); + }); + + it("keeps old snapshots readable when no routing trace exists", async () => { + const snapshot: LiveChainSnapshot = { + chain: [{ id: 1, name: "provider-a", reason: "retry_failed", timestamp: 100 }], + phase: "retrying", + updatedAt: 100, + }; + getStore(CHAIN_PREFIX).data.set("legacy:1", snapshot); + + await expect(readLiveChain("legacy", 1)).resolves.toEqual({ + ...snapshot, + routingTrace: null, + }); + }); + + it("accepts an embedded trace written during a rolling upgrade", async () => { + const trace = makeTrace({ + events: [{ type: "winner_committed", at: 200, elapsedMs: 100 }], + }); + getStore(CHAIN_PREFIX).data.set("rolling:1", { + chain: [], + phase: "queued", + updatedAt: 100, + routingTrace: trace, + } satisfies LiveChainSnapshot); + + await expect(readLiveChain("rolling", 1)).resolves.toMatchObject({ + phase: "streaming", + routingTrace: trace, + }); + }); + + it("ignores a malformed independently stored trace", async () => { + getStore(CHAIN_PREFIX).data.set("malformed:1", { + chain: [], + phase: "queued", + updatedAt: 100, + } satisfies LiveChainSnapshot); + getStore(TRACE_PREFIX).data.set("malformed:1", { + version: 999, + events: [], + }); + + await expect(readLiveChain("malformed", 1)).resolves.toMatchObject({ + phase: "queued", + routingTrace: null, + }); + }); + + it("merges routing traces for batch reads", async () => { + await writeLiveChain("batch", 1, []); + await writeLiveChain("batch", 2, []); + await writeLiveRoutingTrace( + "batch", + 2, + makeTrace({ + events: [{ type: "sticky_probe_started", at: 100, elapsedMs: 0 }], + }) + ); + + const result = await readLiveChainBatch([ + { sessionId: "batch", requestSequence: 1 }, + { sessionId: "batch", requestSequence: 2 }, + { sessionId: "missing", requestSequence: 3 }, + ]); + + expect(result.size).toBe(2); + expect(result.get("batch:1")).toMatchObject({ + phase: "queued", + routingTrace: null, + }); + expect(result.get("batch:2")).toMatchObject({ phase: "discovery_sticky" }); + }); + + it("deletes both live keys", async () => { + await writeLiveChain("session-a", 3, []); + await writeLiveRoutingTrace("session-a", 3, makeTrace()); + + await deleteLiveChain("session-a", 3); + + expect(getStore(CHAIN_PREFIX).data.has("session-a:3")).toBe(false); + expect(getStore(TRACE_PREFIX).data.has("session-a:3")).toBe(false); + expect(getStore(CHAIN_PREFIX).deleted).toContain("session-a:3"); + expect(getStore(TRACE_PREFIX).deleted).toContain("session-a:3"); + }); +}); diff --git a/src/lib/redis/live-chain-store.test.ts b/src/lib/redis/live-chain-store.test.ts index 79b79d1ab..54e3370b9 100644 --- a/src/lib/redis/live-chain-store.test.ts +++ b/src/lib/redis/live-chain-store.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; import { inferPhase } from "./live-chain-store"; import type { ProviderChainItem } from "@/types/message"; +import type { RoutingTraceEventV1, RoutingTraceV1 } from "@/types/routing-trace"; // Note: writeLiveChain/readLiveChain/readLiveChainBatch/deleteLiveChain // require "server-only" + Redis, so they are tested via integration tests. @@ -10,6 +11,22 @@ function makeChainItem(overrides: Partial = {}): ProviderChai return { id: 1, name: "provider-a", timestamp: Date.now(), ...overrides }; } +function makeDiscoveryTrace( + event: RoutingTraceEventV1, + overrides: Partial = {} +): RoutingTraceV1 { + return { + version: 1, + mode: "discovery", + startedAt: 100, + updatedAt: 200, + discoveryEnabled: true, + eligible: true, + events: [event], + ...overrides, + }; +} + describe("inferPhase", () => { it('returns "queued" for empty chain', () => { expect(inferPhase([])).toBe("queued"); @@ -95,4 +112,112 @@ describe("inferPhase", () => { ]; expect(inferPhase(chain)).toBe("streaming"); }); + + it.each([ + ["sticky_probe_started", "discovery_sticky"], + ["round_started", "discovery_racing"], + ["attempt_started", "discovery_racing"], + ["attempt_ready", "discovery_racing"], + ["attempt_held", "discovery_racing"], + ["attempt_finished", "discovery_racing"], + ["sticky_timeout", "discovery_racing"], + ["fallback_promoted", "discovery_fallback"], + ["winner_committed", "streaming"], + ["binding_finalized", "streaming"], + ] as const)("derives %s Discovery events as %s", (type, expected) => { + expect(inferPhase([], makeDiscoveryTrace({ type, at: 100, elapsedMs: 0 }))).toBe(expected); + }); + + it.each([ + ["success", "completed"], + ["failed", "failed"], + ["client_abort", "aborted"], + ["deadline", "deadline"], + ] as const)("derives Discovery terminal outcome %s as %s", (outcome, expected) => { + expect( + inferPhase( + [], + makeDiscoveryTrace( + { type: "request_finished", at: 200, elapsedMs: 100, outcome }, + { + summary: { + outcome, + statusCode: outcome === "success" ? 200 : 503, + durationMs: 100, + ttfbMs: null, + attemptsPerRequest: 2, + maxActiveAttempts: 2, + rounds: 1, + providerMs: 200, + fallbackPromotions: 0, + cancelFailures: 0, + winnerOrigin: outcome === "success" ? "normal" : "none", + winnerProviderId: outcome === "success" ? 1 : null, + winnerRound: outcome === "success" ? 1 : null, + }, + } + ) + ) + ).toBe(expected); + }); + + it("keeps the terminal phase when binding finalization is appended later", () => { + const trace = makeDiscoveryTrace( + { type: "binding_finalized", at: 210, elapsedMs: 110 }, + { + events: [ + { type: "request_finished", at: 200, elapsedMs: 100, outcome: "success" }, + { type: "binding_finalized", at: 210, elapsedMs: 110 }, + ], + summary: { + outcome: "success", + statusCode: 200, + durationMs: 100, + ttfbMs: 20, + attemptsPerRequest: 2, + maxActiveAttempts: 2, + rounds: 1, + providerMs: 200, + fallbackPromotions: 0, + cancelFailures: 0, + winnerOrigin: "normal", + winnerProviderId: 1, + winnerRound: 1, + }, + } + ); + expect(inferPhase([], trace)).toBe("completed"); + }); + + it("keeps a first-byte winner streaming until request_finished exists", () => { + const trace = makeDiscoveryTrace( + { type: "winner_committed", at: 200, elapsedMs: 100 }, + { + summary: { + outcome: "success", + statusCode: 200, + durationMs: 100, + ttfbMs: 20, + attemptsPerRequest: 2, + maxActiveAttempts: 2, + rounds: 1, + providerMs: 200, + fallbackPromotions: 0, + cancelFailures: 0, + winnerOrigin: "normal", + winnerProviderId: 1, + winnerRound: 1, + }, + } + ); + expect(inferPhase([], trace)).toBe("streaming"); + }); + + it("keeps legacy chain phase for a non-Discovery trace", () => { + const trace = makeDiscoveryTrace( + { type: "winner_committed", at: 200, elapsedMs: 100 }, + { mode: "legacy_serial", eligible: false } + ); + expect(inferPhase([makeChainItem({ reason: "retry_failed" })], trace)).toBe("retrying"); + }); }); diff --git a/src/lib/redis/live-chain-store.ts b/src/lib/redis/live-chain-store.ts index 4915d67f0..114545632 100644 --- a/src/lib/redis/live-chain-store.ts +++ b/src/lib/redis/live-chain-store.ts @@ -1,12 +1,14 @@ import "server-only"; import type { ProviderChainItem } from "@/types/message"; +import { normalizeRoutingTrace, type RoutingTraceV1 } from "@/types/routing-trace"; import { RedisKVStore } from "./redis-kv-store"; export interface LiveChainSnapshot { chain: ProviderChainItem[]; phase: string; updatedAt: number; + routingTrace?: RoutingTraceV1 | null; } const SESSION_TTL = Number.parseInt(process.env.SESSION_TTL || "300", 10); @@ -16,11 +18,74 @@ const store = new RedisKVStore({ defaultTtlSeconds: SESSION_TTL, }); +// Routing traces are updated independently from the provider chain. A separate +// key prevents a legacy chain writer from overwriting concurrent trace events. +const routingTraceStore = new RedisKVStore({ + prefix: "cch:live-routing-trace:", + defaultTtlSeconds: SESSION_TTL, +}); + function buildKey(sessionId: string, requestSequence: number): string { return `${sessionId}:${requestSequence}`; } -export function inferPhase(chain: ProviderChainItem[]): string { +function inferDiscoveryPhase(trace: RoutingTraceV1): string { + const terminalEvent = trace.events.findLast((event) => event.type === "request_finished"); + if (terminalEvent) { + switch (trace.summary?.outcome ?? terminalEvent.outcome) { + case "success": + return "completed"; + case "client_abort": + return "aborted"; + case "deadline": + return "deadline"; + case "failed": + return "failed"; + } + } + + const last = trace.events[trace.events.length - 1]; + if (!last) return "discovery_racing"; + + switch (last.type) { + case "sticky_probe_started": + return "discovery_sticky"; + case "fallback_promoted": + return "discovery_fallback"; + case "winner_committed": + case "binding_finalized": + return "streaming"; + case "request_finished": + switch (last.outcome) { + case "success": + return "completed"; + case "client_abort": + return "aborted"; + case "deadline": + return "deadline"; + default: + return "failed"; + } + case "round_started": + case "sticky_timeout": + case "attempt_started": + case "attempt_ready": + case "attempt_held": + case "attempt_finished": + return "discovery_racing"; + } + + return "discovery_racing"; +} + +export function inferPhase( + chain: ProviderChainItem[], + routingTrace?: RoutingTraceV1 | null +): string { + if (routingTrace?.mode === "discovery") { + return inferDiscoveryPhase(routingTrace); + } + if (chain.length === 0) return "queued"; const last = chain[chain.length - 1]; switch (last.reason) { @@ -49,6 +114,35 @@ export function inferPhase(chain: ProviderChainItem[]): string { } } +function mergeSnapshot( + snapshot: LiveChainSnapshot | null, + storedRoutingTrace: unknown +): LiveChainSnapshot | null { + // During rolling upgrades a trace may already be embedded in the old + // snapshot shape. Prefer the independently updated trace when both exist. + const routingTrace = + normalizeRoutingTrace(storedRoutingTrace) ?? normalizeRoutingTrace(snapshot?.routingTrace); + + // Trace recording starts before provider selection can append to the legacy + // chain. Keep that earliest Discovery state visible instead of waiting for a + // second Redis write. + if (!snapshot) { + if (!routingTrace) return null; + return { + chain: [], + phase: inferPhase([], routingTrace), + updatedAt: routingTrace.updatedAt, + routingTrace, + }; + } + + return { + ...snapshot, + phase: inferPhase(snapshot.chain, routingTrace), + routingTrace, + }; +} + export async function writeLiveChain( sessionId: string, requestSequence: number, @@ -62,11 +156,21 @@ export async function writeLiveChain( await store.set(buildKey(sessionId, requestSequence), snapshot); } +export async function writeLiveRoutingTrace( + sessionId: string, + requestSequence: number, + routingTrace: RoutingTraceV1 +): Promise { + await routingTraceStore.set(buildKey(sessionId, requestSequence), routingTrace); +} + export async function readLiveChain( sessionId: string, requestSequence: number ): Promise { - return store.get(buildKey(sessionId, requestSequence)); + const key = buildKey(sessionId, requestSequence); + const [snapshot, routingTrace] = await Promise.all([store.get(key), routingTraceStore.get(key)]); + return mergeSnapshot(snapshot, routingTrace); } export async function readLiveChainBatch( @@ -77,8 +181,12 @@ export async function readLiveChainBatch( const entries = await Promise.all( keys.map(async (k) => { - const snapshot = await store.get(buildKey(k.sessionId, k.requestSequence)); - return { key: buildKey(k.sessionId, k.requestSequence), snapshot }; + const key = buildKey(k.sessionId, k.requestSequence); + const [snapshot, routingTrace] = await Promise.all([ + store.get(key), + routingTraceStore.get(key), + ]); + return { key, snapshot: mergeSnapshot(snapshot, routingTrace) }; }) ); @@ -89,5 +197,6 @@ export async function readLiveChainBatch( } export async function deleteLiveChain(sessionId: string, requestSequence: number): Promise { - await store.delete(buildKey(sessionId, requestSequence)); + const key = buildKey(sessionId, requestSequence); + await Promise.all([store.delete(key), routingTraceStore.delete(key)]); } diff --git a/src/repository/_shared/transformers.ts b/src/repository/_shared/transformers.ts index 568a50ba7..b91beb78f 100644 --- a/src/repository/_shared/transformers.ts +++ b/src/repository/_shared/transformers.ts @@ -5,6 +5,7 @@ import type { Key } from "@/types/key"; import type { MessageRequest } from "@/types/message"; import type { ModelPrice } from "@/types/model-price"; import type { Provider } from "@/types/provider"; +import { normalizeRoutingTrace } from "@/types/routing-trace"; import { DEFAULT_FAKE_STREAMING_WHITELIST, type FakeStreamingWhitelistEntry, @@ -183,6 +184,7 @@ export function toMessageRequest(dbMessage: any): MessageRequest { context1mApplied: dbMessage?.context1mApplied ?? false, swapCacheTtlApplied: dbMessage?.swapCacheTtlApplied ?? false, specialSettings: dbMessage?.specialSettings ?? null, + routingTrace: normalizeRoutingTrace(dbMessage?.routingTrace), }; } diff --git a/src/repository/message-write-buffer.ts b/src/repository/message-write-buffer.ts index e58f871e8..ce6c3309f 100644 --- a/src/repository/message-write-buffer.ts +++ b/src/repository/message-write-buffer.ts @@ -8,6 +8,7 @@ import { getEnvConfig } from "@/lib/config/env.schema"; import { logger } from "@/lib/logger"; import type { StoredCostBreakdown } from "@/types/cost-breakdown"; import type { CreateMessageRequestData } from "@/types/message"; +import { normalizeRoutingTrace, type RoutingTraceV1 } from "@/types/routing-trace"; export type MessageRequestUpdatePatch = { durationMs?: number; @@ -22,6 +23,7 @@ export type MessageRequestUpdatePatch = { cacheCreation1hInputTokens?: number; cacheTtlApplied?: string | null; providerChain?: CreateMessageRequestData["provider_chain"]; + routingTrace?: RoutingTraceV1 | null; errorMessage?: string; errorStack?: string; errorCause?: string; @@ -231,6 +233,7 @@ const COLUMN_MAP: Record = { cacheCreation1hInputTokens: "cache_creation_1h_input_tokens", cacheTtlApplied: "cache_ttl_applied", providerChain: "provider_chain", + routingTrace: "routing_trace", errorMessage: "error_message", errorStack: "error_stack", errorCause: "error_cause", @@ -297,12 +300,17 @@ export function buildBatchUpdateSql( continue; } - if (key === "providerChain" || key === "specialSettings" || key === "costBreakdown") { + if ( + key === "providerChain" || + key === "routingTrace" || + key === "specialSettings" || + key === "costBreakdown" + ) { if (value === null) { cases.push(sql`WHEN ${update.id} THEN NULL`); continue; } - const json = JSON.stringify(value); + const json = JSON.stringify(key === "routingTrace" ? normalizeRoutingTrace(value) : value); cases.push(sql`WHEN ${update.id} THEN ${json}::jsonb`); continue; } diff --git a/src/repository/message.ts b/src/repository/message.ts index 4d72a6944..c12167f7c 100644 --- a/src/repository/message.ts +++ b/src/repository/message.ts @@ -13,6 +13,7 @@ import { import { formatCostForStorage } from "@/lib/utils/currency"; import type { HedgeLoserBilling, StoredCostBreakdown } from "@/types/cost-breakdown"; import type { CreateMessageRequestData, MessageRequest, ProviderChainItem } from "@/types/message"; +import { normalizeRoutingTrace, type RoutingTraceV1 } from "@/types/routing-trace"; import type { SpecialSetting } from "@/types/special-settings"; import { LEDGER_BILLING_CONDITION } from "./_shared/ledger-conditions"; import { EXCLUDE_WARMUP_CONDITION } from "./_shared/message-request-conditions"; @@ -238,6 +239,8 @@ export async function createMessageRequest( groupCostMultiplier: data.group_cost_multiplier?.toString() ?? undefined, // 分组倍率（转为字符串） sessionId: data.session_id, // Session ID requestSequence: data.request_sequence, // Request Sequence（Session 内请求序号） + routingTrace: + data.routing_trace === undefined ? undefined : normalizeRoutingTrace(data.routing_trace), userAgent: data.user_agent, // User-Agent clientIp: data.client_ip, // 客户端 IP（IPv4/IPv6） endpoint: data.endpoint, // 请求端点（可为空） @@ -262,6 +265,7 @@ export async function createMessageRequest( costMultiplier: messageRequest.costMultiplier, // 新增 sessionId: messageRequest.sessionId, // 新增 requestSequence: messageRequest.requestSequence, // Request Sequence + routingTrace: messageRequest.routingTrace, userAgent: messageRequest.userAgent, // 新增 clientIp: messageRequest.clientIp, // 客户端 IP endpoint: messageRequest.endpoint, // 新增：返回端点 @@ -488,6 +492,7 @@ export type MessageRequestDetailsUpdate = { cacheCreation1hInputTokens?: number; cacheTtlApplied?: string | null; providerChain?: CreateMessageRequestData["provider_chain"]; + routingTrace?: RoutingTraceV1 | null; errorMessage?: string; errorStack?: string; // 完整堆栈信息 errorCause?: string; // 嵌套错误原因（JSON 格式） @@ -555,6 +560,9 @@ export async function updateMessageRequestDetails( if (details.providerChain !== undefined) { updateData.providerChain = details.providerChain; } + if (details.routingTrace !== undefined) { + updateData.routingTrace = normalizeRoutingTrace(details.routingTrace); + } if (details.errorMessage !== undefined) { updateData.errorMessage = details.errorMessage; } @@ -606,6 +614,36 @@ export async function updateMessageRequestDetails( return true; } +/** + * Best-effort routing trace patch for work that completes after the request's + * terminal row has been committed (for example, Sticky binding finalization). + * This intentionally bypasses terminal ownership and public-status rollups. + */ +export async function updateMessageRequestRoutingTrace( + id: number, + routingTrace: RoutingTraceV1 +): Promise { + const normalized = normalizeRoutingTrace(routingTrace); + if (!normalized) return; + + if (getEnvConfig().MESSAGE_REQUEST_WRITE_MODE === "async") { + enqueueMessageRequestUpdate(id, { routingTrace: normalized }); + return; + } + + try { + await db + .update(messageRequest) + .set({ routingTrace: normalized, updatedAt: new Date() }) + .where(and(eq(messageRequest.id, id), isNull(messageRequest.deletedAt))); + } catch (error) { + logger.warn("[MessageRequest] Failed to patch finalized routing trace", { + requestId: id, + error: error instanceof Error ? error.message : String(error), + }); + } +} + export async function updateMessageRequestDetailsIfUnfinalized( id: number, details: MessageRequestDetailsUpdate, @@ -743,6 +781,7 @@ export async function findMessageRequestById(id: number): Promise([ + "discovery", + "legacy_hedge", + "legacy_serial", + "single_upstream", +]); +const ROUTING_TRACE_EVENT_TYPES = new Set([ + "request_started", + "round_started", + "sticky_probe_started", + "sticky_timeout", + "attempt_started", + "attempt_ready", + "attempt_held", + "attempt_finished", + "fallback_promoted", + "winner_committed", + "binding_finalized", + "request_finished", +]); + +function finiteNumber(value: unknown): number | undefined { + return typeof value === "number" && Number.isFinite(value) ? value : undefined; +} + +function nonEmptyString(value: unknown): string | undefined { + return typeof value === "string" && value.length > 0 ? value : undefined; +} + +function normalizeRoutingTraceEvent(value: unknown): RoutingTraceEventV1 | null { + if (!value || typeof value !== "object" || Array.isArray(value)) return null; + const event = value as Record; + if ( + !ROUTING_TRACE_EVENT_TYPES.has(event.type as RoutingTraceEventType) || + finiteNumber(event.at) === undefined || + finiteNumber(event.elapsedMs) === undefined + ) { + return null; + } + + const attemptKinds = new Set(["sticky", "normal", "fallback"]); + const bindingActions = new Set>([ + "create", + "renew", + "clear", + "none", + ]); + const providerValue = event.provider; + const provider = + providerValue && typeof providerValue === "object" && !Array.isArray(providerValue) + ? (providerValue as Record) + : null; + const providerId = finiteNumber(provider?.id); + + return { + type: event.type as RoutingTraceEventType, + at: event.at as number, + elapsedMs: event.elapsedMs as number, + ...(finiteNumber(event.round) !== undefined ? { round: event.round as number } : {}), + ...(nonEmptyString(event.attemptId) ? { attemptId: event.attemptId as string } : {}), + ...(attemptKinds.has(event.attemptKind as RoutingTraceAttemptKind) + ? { attemptKind: event.attemptKind as RoutingTraceAttemptKind } + : {}), + ...(providerId !== undefined + ? { + provider: { + id: providerId, + ...(nonEmptyString(provider?.name) ? { name: provider?.name as string } : {}), + ...(finiteNumber(provider?.priority) !== undefined + ? { priority: provider?.priority as number } + : {}), + }, + } + : {}), + ...(nonEmptyString(event.outcome) ? { outcome: event.outcome as string } : {}), + ...(nonEmptyString(event.cancellationKind) + ? { cancellationKind: event.cancellationKind as string } + : {}), + ...(finiteNumber(event.statusCode) !== undefined + ? { statusCode: event.statusCode as number } + : {}), + ...(nonEmptyString(event.reason) ? { reason: event.reason as string } : {}), + ...(bindingActions.has(event.bindingAction as NonNullable) + ? { + bindingAction: event.bindingAction as NonNullable, + } + : {}), + ...(finiteNumber(event.durationMs) !== undefined + ? { durationMs: event.durationMs as number } + : {}), + }; +} + +function normalizeRoutingTraceConfig(value: unknown): RoutingTraceConfigV1 | undefined { + if (!value || typeof value !== "object" || Array.isArray(value)) return undefined; + const config = value as Record; + const discoveryConcurrency = finiteNumber(config.discoveryConcurrency); + const maxDiscoveryRounds = finiteNumber(config.maxDiscoveryRounds); + const discoverySlaMs = finiteNumber(config.discoverySlaMs); + const stickySlaMs = finiteNumber(config.stickySlaMs); + const racingTotalTimeoutMs = finiteNumber(config.racingTotalTimeoutMs); + const stickyTimeoutCooldownMs = finiteNumber(config.stickyTimeoutCooldownMs); + if ( + discoveryConcurrency === undefined || + maxDiscoveryRounds === undefined || + discoverySlaMs === undefined || + stickySlaMs === undefined || + racingTotalTimeoutMs === undefined || + stickyTimeoutCooldownMs === undefined + ) { + return undefined; + } + return { + discoveryConcurrency, + maxDiscoveryRounds, + discoverySlaMs, + stickySlaMs, + racingTotalTimeoutMs, + stickyTimeoutCooldownMs, + }; +} + +function normalizeRoutingTraceSummary(value: unknown): RoutingTraceSummaryV1 | undefined { + if (!value || typeof value !== "object" || Array.isArray(value)) return undefined; + const summary = value as Record; + const outcomes = new Set([ + "success", + "failed", + "client_abort", + "deadline", + ]); + const winnerOrigins = new Set(["sticky", "normal", "fallback", "none"]); + const numericKeys = [ + "statusCode", + "durationMs", + "attemptsPerRequest", + "maxActiveAttempts", + "rounds", + "providerMs", + "fallbackPromotions", + "cancelFailures", + ] as const; + if ( + !outcomes.has(summary.outcome as RoutingTraceRequestOutcome) || + !winnerOrigins.has(summary.winnerOrigin as RoutingTraceWinnerOrigin) || + numericKeys.some((key) => finiteNumber(summary[key]) === undefined) || + !(summary.ttfbMs === null || finiteNumber(summary.ttfbMs) !== undefined) || + !(summary.winnerProviderId === null || finiteNumber(summary.winnerProviderId) !== undefined) || + !(summary.winnerRound === null || finiteNumber(summary.winnerRound) !== undefined) + ) { + return undefined; + } + return { + outcome: summary.outcome as RoutingTraceRequestOutcome, + statusCode: summary.statusCode as number, + durationMs: summary.durationMs as number, + ttfbMs: summary.ttfbMs as number | null, + attemptsPerRequest: summary.attemptsPerRequest as number, + maxActiveAttempts: summary.maxActiveAttempts as number, + rounds: summary.rounds as number, + providerMs: summary.providerMs as number, + fallbackPromotions: summary.fallbackPromotions as number, + cancelFailures: summary.cancelFailures as number, + winnerOrigin: summary.winnerOrigin as RoutingTraceWinnerOrigin, + winnerProviderId: summary.winnerProviderId as number | null, + winnerRound: summary.winnerRound as number | null, + }; +} + +/** Treat persisted JSON as untrusted so legacy or malformed rows remain readable. */ +export function normalizeRoutingTrace(value: unknown): RoutingTraceV1 | null { + if (!value || typeof value !== "object" || Array.isArray(value)) return null; + + const trace = value as Partial; + if ( + trace.version !== ROUTING_TRACE_VERSION || + !ROUTING_TRACE_MODES.has(trace.mode as RoutingTraceMode) || + typeof trace.startedAt !== "number" || + !Number.isFinite(trace.startedAt) || + typeof trace.updatedAt !== "number" || + !Number.isFinite(trace.updatedAt) || + typeof trace.discoveryEnabled !== "boolean" || + typeof trace.eligible !== "boolean" || + !Array.isArray(trace.events) + ) { + return null; + } + + const events = trace.events + .slice(0, ROUTING_TRACE_MAX_EVENTS) + .map(normalizeRoutingTraceEvent) + .filter((event): event is RoutingTraceEventV1 => event !== null); + const config = normalizeRoutingTraceConfig(trace.config); + const summary = normalizeRoutingTraceSummary(trace.summary); + + return { + version: ROUTING_TRACE_VERSION, + mode: trace.mode as RoutingTraceMode, + startedAt: trace.startedAt, + updatedAt: trace.updatedAt, + discoveryEnabled: trace.discoveryEnabled, + eligible: trace.eligible, + ...(nonEmptyString(trace.bypassReason) ? { bypassReason: trace.bypassReason } : {}), + ...(config ? { config } : {}), + events, + ...(summary ? { summary } : {}), + ...(trace.truncated === true || trace.events.length > ROUTING_TRACE_MAX_EVENTS + ? { truncated: true } + : {}), + }; +} diff --git a/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts b/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts index 436693834..76ddca059 100644 --- a/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts +++ b/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts @@ -461,7 +461,7 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { Date.now() ); - expect(prepared).not.toBeNull(); + expect(prepared).toMatchObject({ status: "prepared" }); expect(mocks.ensureVersionedBindingCapability).toHaveBeenCalledTimes(1); expect(mocks.getSessionBindingSnapshot).toHaveBeenCalledWith("sess-hedge", 20); expect(mocks.acquireSessionDiscoveryLease).toHaveBeenCalledTimes(1); @@ -504,7 +504,10 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { Date.now() ); - expect(prepared).toBeNull(); + expect(prepared).toEqual({ + status: "skipped", + reason: "redis_capability_unavailable", + }); expect(mocks.ensureVersionedBindingCapability).toHaveBeenCalledTimes(1); expect(mocks.getSessionBindingSnapshot).not.toHaveBeenCalled(); expect(mocks.acquireSessionDiscoveryLease).not.toHaveBeenCalled(); diff --git a/tests/unit/proxy/response-handler-endpoint-circuit-isolation.test.ts b/tests/unit/proxy/response-handler-endpoint-circuit-isolation.test.ts index 947bfdf80..1de1404bb 100644 --- a/tests/unit/proxy/response-handler-endpoint-circuit-isolation.test.ts +++ b/tests/unit/proxy/response-handler-endpoint-circuit-isolation.test.ts @@ -1186,6 +1186,11 @@ describe("Endpoint circuit breaker isolation", () => { }, ])("fails binding closed when the finalizer lease is $label", async ({ leaseResult }) => { const session = createSession(); + const appendRoutingTraceEvent = vi.fn(); + Object.assign(session, { + appendRoutingTraceEvent, + getRoutingTrace: () => null, + }); session.recordProviderSessionRef(1); setDeferredStreamingFinalization(session, { providerId: 1, @@ -1227,6 +1232,66 @@ describe("Endpoint circuit breaker isolation", () => { expect(SessionManager.compareAndSetSessionProvider).not.toHaveBeenCalled(); expect(RateLimitService.releaseProviderSession).toHaveBeenCalledWith(1, "fake-session"); expect(SessionManager.releaseSessionDiscoveryLease).toHaveBeenCalledOnce(); + expect(appendRoutingTraceEvent).toHaveBeenCalledWith( + expect.objectContaining({ + type: "binding_finalized", + bindingAction: "create", + outcome: "skipped", + reason: "discovery_lease_not_owned", + }) + ); + expect(appendRoutingTraceEvent).not.toHaveBeenCalledWith( + expect.objectContaining({ reason: "binding_not_permitted" }) + ); + }); + + it("records a missing Discovery binding snapshot instead of a generic denial", async () => { + const session = createSession(); + const appendRoutingTraceEvent = vi.fn(); + Object.assign(session, { + appendRoutingTraceEvent, + getRoutingTrace: () => null, + }); + setDeferredStreamingFinalization(session, { + providerId: 1, + providerName: "test-provider", + providerPriority: 10, + attemptNumber: 1, + totalProvidersAttempted: 2, + isFirstAttempt: false, + isFailoverSuccess: true, + endpointId: 42, + endpointUrl: "https://api.test.com", + upstreamStatusCode: 200, + bindingIntent: "create", + requiresCompletionMarker: true, + discoveryLease: { + sessionId: "fake-session", + keyId: 456, + ownerToken: "missing-snapshot-owner", + ttlSeconds: 30, + }, + }); + + const clientResponse = await ProxyResponseHandler.dispatch( + session, + createSuccessStreamResponseWithCompletion() + ); + await clientResponse.text(); + await drainAsyncTasks(); + + expect(SessionManager.compareAndSetSessionProvider).not.toHaveBeenCalled(); + expect(appendRoutingTraceEvent).toHaveBeenCalledWith( + expect.objectContaining({ + type: "binding_finalized", + bindingAction: "create", + outcome: "skipped", + reason: "missing_snapshot", + }) + ); + expect(appendRoutingTraceEvent).not.toHaveBeenCalledWith( + expect.objectContaining({ reason: "binding_not_permitted" }) + ); }); it("renews a long-stream lease and releases it once after terminal side effects", async () => { diff --git a/tests/unit/proxy/routing-trace.test.ts b/tests/unit/proxy/routing-trace.test.ts new file mode 100644 index 000000000..a6c1dcacc --- /dev/null +++ b/tests/unit/proxy/routing-trace.test.ts @@ -0,0 +1,353 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { resolveEndpointPolicy } from "@/app/v1/_lib/proxy/endpoint-policy"; +import { normalizeRoutingTrace, ROUTING_TRACE_MAX_EVENTS } from "@/types/routing-trace"; +import type { SystemSettings } from "@/types/system-config"; + +const liveChainMocks = vi.hoisted(() => ({ + deleteLiveChain: vi.fn(async () => undefined), + writeLiveChain: vi.fn(async () => undefined), + writeLiveRoutingTrace: vi.fn(async () => undefined), +})); + +vi.mock("@/lib/redis/live-chain-store", () => liveChainMocks); + +vi.mock("@/lib/logger", () => ({ + logger: { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + trace: vi.fn(), + error: vi.fn(), + fatal: vi.fn(), + }, +})); + +import { ProxyForwarder } from "@/app/v1/_lib/proxy/forwarder"; +import { ProxySession } from "@/app/v1/_lib/proxy/session"; +import { logger } from "@/lib/logger"; + +type PrepareStreamingDiscovery = ( + session: ProxySession, + settings: SystemSettings, + requestStartedAt: number +) => Promise<{ status: "prepared"; prepared: unknown } | { status: "skipped"; reason: string }>; + +function prepareStreamingDiscovery(): PrepareStreamingDiscovery { + return ( + ProxyForwarder as unknown as { + prepareStreamingDiscovery: PrepareStreamingDiscovery; + } + ).prepareStreamingDiscovery; +} + +function makePreparationSession(stream: boolean): ProxySession { + return { + request: { message: { stream } }, + originalFormat: "claude", + getEndpointPolicy: () => resolveEndpointPolicy("/v1/messages"), + } as unknown as ProxySession; +} + +function makeTraceSession(startTime = 1_000): ProxySession { + const session = Object.create(ProxySession.prototype) as ProxySession; + Object.assign(session, { + startTime, + sessionId: "routing-trace-session", + requestSequence: 7, + highConcurrencyModeEnabled: false, + routingTrace: null, + liveChainDirty: false, + liveRoutingTraceDirty: false, + liveObservabilityFlushPromise: null, + liveObservabilityClosePromise: null, + liveObservabilityClosed: false, + routingTraceTerminalLogged: false, + providerChain: [], + ttfbMs: null, + }); + return session; +} + +describe("routing trace Discovery preparation", () => { + it("reports disabled before touching request-specific eligibility", async () => { + const result = await prepareStreamingDiscovery()( + {} as ProxySession, + { discoveryEnabled: false } as SystemSettings, + 1_000 + ); + + expect(result).toEqual({ status: "skipped", reason: "disabled" }); + }); + + it("reports non_streaming for an otherwise supported endpoint and protocol", async () => { + const result = await prepareStreamingDiscovery()( + makePreparationSession(false), + { discoveryEnabled: true } as SystemSettings, + 1_000 + ); + + expect(result).toEqual({ status: "skipped", reason: "non_streaming" }); + }); +}); + +describe("ProxySession routing trace recorder", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("preserves a focused Discovery lifecycle order through terminal finalization", () => { + const session = makeTraceSession(); + session.initializeRoutingTrace({ + mode: "discovery", + discoveryEnabled: true, + eligible: true, + startedAt: 1_000, + }); + + session.appendRoutingTraceEvent({ + type: "round_started", + round: 1, + at: 1_010, + }); + session.appendRoutingTraceEvent({ + type: "attempt_started", + round: 1, + attemptId: "attempt-1", + attemptKind: "normal", + provider: { id: 11, name: "Provider 11", priority: 1 }, + at: 1_020, + }); + session.appendRoutingTraceEvent({ + type: "attempt_ready", + round: 1, + attemptId: "attempt-1", + attemptKind: "normal", + provider: { id: 11, name: "Provider 11", priority: 1 }, + at: 1_030, + }); + session.appendRoutingTraceEvent({ + type: "attempt_finished", + round: 1, + attemptId: "attempt-1", + attemptKind: "normal", + provider: { id: 11, name: "Provider 11", priority: 1 }, + outcome: "winner", + at: 1_040, + }); + session.appendRoutingTraceEvent({ + type: "winner_committed", + round: 1, + attemptId: "attempt-1", + attemptKind: "normal", + provider: { id: 11, name: "Provider 11", priority: 1 }, + statusCode: 200, + at: 1_050, + }); + session.finalizeRoutingTrace(200, "success"); + + expect(session.getRoutingTrace()?.events.map((event) => event.type)).toEqual([ + "request_started", + "round_started", + "attempt_started", + "attempt_ready", + "attempt_finished", + "winner_committed", + "request_finished", + ]); + expect(session.getRoutingTrace()?.events.at(-1)).toMatchObject({ + type: "request_finished", + outcome: "success", + statusCode: 200, + }); + }); + + it("caps the trace at 512 events and persists the truncated snapshot independently", async () => { + const session = makeTraceSession(); + session.initializeRoutingTrace({ + mode: "discovery", + discoveryEnabled: true, + eligible: true, + startedAt: 1_000, + }); + + for (let index = 1; index <= ROUTING_TRACE_MAX_EVENTS; index += 1) { + session.appendRoutingTraceEvent({ + type: "attempt_started", + attemptId: `attempt-${index}`, + attemptKind: "normal", + provider: { id: index }, + at: 1_000 + index, + }); + } + await vi.waitFor(() => expect(liveChainMocks.writeLiveRoutingTrace).toHaveBeenCalled()); + const writesAtLimit = liveChainMocks.writeLiveRoutingTrace.mock.calls.length; + session.appendRoutingTraceEvent({ + type: "attempt_started", + attemptId: "ignored-after-limit", + attemptKind: "normal", + provider: { id: 999 }, + at: 2_000, + }); + await Promise.resolve(); + expect(liveChainMocks.writeLiveRoutingTrace).toHaveBeenCalledTimes(writesAtLimit); + await session.closeLiveObservability(); + + const trace = session.getRoutingTrace(); + expect(trace?.events).toHaveLength(ROUTING_TRACE_MAX_EVENTS); + expect(trace?.truncated).toBe(true); + expect(liveChainMocks.writeLiveRoutingTrace).toHaveBeenCalled(); + expect(liveChainMocks.writeLiveRoutingTrace).toHaveBeenLastCalledWith( + "routing-trace-session", + 7, + expect.objectContaining({ + events: expect.arrayContaining([ + expect.objectContaining({ type: "request_started" }), + expect.objectContaining({ attemptId: "attempt-511" }), + ]), + truncated: true, + }) + ); + expect(liveChainMocks.deleteLiveChain).toHaveBeenCalledWith("routing-trace-session", 7); + }); + + it("coalesces events produced while a live trace write is in flight", async () => { + let releaseFirstWrite!: () => void; + const firstWrite = new Promise((resolve) => { + releaseFirstWrite = resolve; + }); + liveChainMocks.writeLiveRoutingTrace + .mockImplementationOnce(async () => firstWrite) + .mockResolvedValue(undefined); + + const session = makeTraceSession(); + session.initializeRoutingTrace({ + mode: "discovery", + discoveryEnabled: true, + eligible: true, + startedAt: 1_000, + }); + await vi.waitFor(() => expect(liveChainMocks.writeLiveRoutingTrace).toHaveBeenCalledTimes(1)); + + for (let index = 1; index <= 100; index += 1) { + session.appendRoutingTraceEvent({ + type: "attempt_started", + attemptId: `attempt-${index}`, + attemptKind: "normal", + provider: { id: index }, + at: 1_000 + index, + }); + } + const closePromise = session.closeLiveObservability(); + releaseFirstWrite(); + await closePromise; + + expect(liveChainMocks.writeLiveRoutingTrace).toHaveBeenCalledTimes(2); + expect(liveChainMocks.writeLiveRoutingTrace).toHaveBeenLastCalledWith( + "routing-trace-session", + 7, + expect.objectContaining({ + events: expect.arrayContaining([expect.objectContaining({ attemptId: "attempt-100" })]), + }) + ); + }); + + it("keeps winner metrics request-local and logs only the final retried terminal outcome", async () => { + const session = makeTraceSession(); + session.initializeRoutingTrace({ + mode: "discovery", + discoveryEnabled: true, + eligible: true, + startedAt: 1_000, + }); + session.setRoutingTraceSummary({ + outcome: "success", + statusCode: 200, + durationMs: 50, + ttfbMs: 50, + attemptsPerRequest: 2, + maxActiveAttempts: 2, + rounds: 1, + providerMs: 80, + fallbackPromotions: 0, + cancelFailures: 0, + winnerOrigin: "normal", + winnerProviderId: 11, + winnerRound: 1, + }); + + expect(session.getRoutingTrace()?.summary).toBeUndefined(); + session.finalizeRoutingTrace(200, "success"); + session.finalizeRoutingTrace(502, "failed"); + + expect(session.getRoutingTrace()?.summary).toMatchObject({ + outcome: "failed", + statusCode: 502, + }); + expect( + session.getRoutingTrace()?.events.filter((event) => event.type === "request_finished") + ).toEqual([ + expect.objectContaining({ type: "request_finished", outcome: "failed", statusCode: 502 }), + ]); + await session.closeLiveObservability(); + expect(logger.info).toHaveBeenCalledTimes(1); + expect(logger.info).toHaveBeenCalledWith( + "[DiscoveryMetric] Request aggregate", + expect.objectContaining({ + event: "request_finished", + outcome: "failed", + statusCode: 502, + }) + ); + }); +}); + +describe("routing trace sanitization", () => { + it("drops request bodies, keys, upstream URLs and raw error payloads", () => { + const normalized = normalizeRoutingTrace({ + version: 1, + mode: "discovery", + startedAt: 1_000, + updatedAt: 1_010, + discoveryEnabled: true, + eligible: true, + requestBody: { prompt: "secret" }, + apiKey: "sk-secret", + events: [ + { + type: "attempt_finished", + at: 1_010, + elapsedMs: 10, + provider: { + id: 11, + name: "Provider 11", + endpointUrl: "https://secret.example.test/v1/messages", + }, + rawErrorBody: { error: "secret" }, + requestBody: "secret", + apiKey: "sk-secret", + outcome: "failed", + reason: "provider_error", + }, + ], + }); + + expect(normalized).toEqual({ + version: 1, + mode: "discovery", + startedAt: 1_000, + updatedAt: 1_010, + discoveryEnabled: true, + eligible: true, + events: [ + { + type: "attempt_finished", + at: 1_010, + elapsedMs: 10, + provider: { id: 11, name: "Provider 11" }, + outcome: "failed", + reason: "provider_error", + }, + ], + }); + }); +}); diff --git a/tests/unit/proxy/terminal-outcome-contract.test.ts b/tests/unit/proxy/terminal-outcome-contract.test.ts index 8a9d1f859..d41526018 100644 --- a/tests/unit/proxy/terminal-outcome-contract.test.ts +++ b/tests/unit/proxy/terminal-outcome-contract.test.ts @@ -220,6 +220,8 @@ describe("terminal outcome contract", () => { getContext1mApplied: () => false, getGroupCostMultiplier: () => 1, getSpecialSettings: () => null, + finalizeRoutingTrace: () => null, + closeLiveObservability: vi.fn(async () => undefined), } as ProxySession; const handlePromise = ProxyErrorHandler.handle(session, new Error("top-level failure")); diff --git a/tests/unit/repository/message-terminal-write-apis.test.ts b/tests/unit/repository/message-terminal-write-apis.test.ts index cf060c3a2..5dbe731f8 100644 --- a/tests/unit/repository/message-terminal-write-apis.test.ts +++ b/tests/unit/repository/message-terminal-write-apis.test.ts @@ -179,4 +179,30 @@ describe("message terminal write APIs", () => { expect(updateSet).toHaveBeenCalledWith({ ...details, updatedAt: expect.any(Date) }); expect(updateSet.mock.calls[0]?.[0]).not.toHaveProperty("statusCode"); }); + + it("patches a finalized routing trace without touching terminal or billing fields", async () => { + vi.resetModules(); + const { updateSet, updateWhere } = installSyncBoundaries(); + const { updateMessageRequestRoutingTrace } = await import("@/repository/message"); + const routingTrace = { + version: 1 as const, + mode: "legacy_serial" as const, + startedAt: 1_000, + updatedAt: 1_050, + discoveryEnabled: true, + eligible: false, + bypassReason: "non_streaming", + events: [], + }; + + await updateMessageRequestRoutingTrace(706, routingTrace); + + expect(updateSet).toHaveBeenCalledWith({ + routingTrace, + updatedAt: expect.any(Date), + }); + expect(updateSet.mock.calls[0]?.[0]).not.toHaveProperty("statusCode"); + expect(updateSet.mock.calls[0]?.[0]).not.toHaveProperty("costUsd"); + expect(updateWhere).toHaveBeenCalledTimes(1); + }); }); diff --git a/tests/unit/repository/message-write-buffer.test.ts b/tests/unit/repository/message-write-buffer.test.ts index cdac4fc6c..8602dc78e 100644 --- a/tests/unit/repository/message-write-buffer.test.ts +++ b/tests/unit/repository/message-write-buffer.test.ts @@ -996,6 +996,46 @@ describe("message_request 异步批量写入", () => { expect(built.sql).toContain("::numeric"); expect(built.sql).not.toContain("COALESCE"); }); + + it("routingTrace 应作为 jsonb 写入且保留终态摘要", async () => { + process.env.MESSAGE_REQUEST_WRITE_MODE = "async"; + + const { enqueueMessageRequestUpdate, stopMessageRequestWriteBuffer } = await import( + "@/repository/message-write-buffer" + ); + const routingTrace = { + version: 1 as const, + mode: "discovery" as const, + startedAt: 1_000, + updatedAt: 1_100, + discoveryEnabled: true, + eligible: true, + events: [{ type: "request_finished" as const, at: 1_100, elapsedMs: 100 }], + summary: { + outcome: "success" as const, + statusCode: 200, + durationMs: 100, + ttfbMs: 50, + attemptsPerRequest: 2, + maxActiveAttempts: 2, + rounds: 1, + providerMs: 180, + fallbackPromotions: 0, + cancelFailures: 0, + winnerOrigin: "normal" as const, + winnerProviderId: 7, + winnerRound: 1, + }, + }; + + enqueueMessageRequestUpdate(12, { routingTrace }); + await stopMessageRequestWriteBuffer(); + + const built = toSqlText(executeMock.mock.calls[0]?.[0]); + expect(built.sql).toContain('"routing_trace" = CASE id'); + expect(built.sql).toContain("::jsonb"); + expect(built.params).toContain(JSON.stringify(routingTrace)); + }); }); describe("mergePatch（替换合并语义）", () => { diff --git a/tests/unit/types/routing-trace.test.ts b/tests/unit/types/routing-trace.test.ts new file mode 100644 index 000000000..3c5456452 --- /dev/null +++ b/tests/unit/types/routing-trace.test.ts @@ -0,0 +1,107 @@ +import { describe, expect, it } from "vitest"; +import { normalizeRoutingTrace, ROUTING_TRACE_MAX_EVENTS } from "@/types/routing-trace"; + +describe("normalizeRoutingTrace", () => { + it("未知版本或缺少必要字段时返回 null", () => { + expect(normalizeRoutingTrace(null)).toBeNull(); + expect(normalizeRoutingTrace({ version: 2, events: [] })).toBeNull(); + expect( + normalizeRoutingTrace({ + version: 1, + mode: "discovery", + startedAt: 1, + updatedAt: 2, + discoveryEnabled: true, + eligible: true, + }) + ).toBeNull(); + }); + + it("只保留事件白名单字段并剔除敏感扩展数据", () => { + const trace = normalizeRoutingTrace({ + version: 1, + mode: "discovery", + startedAt: 1, + updatedAt: 2, + discoveryEnabled: true, + eligible: true, + config: { + discoveryConcurrency: 2, + maxDiscoveryRounds: 2, + discoverySlaMs: 10_000, + stickySlaMs: 20_000, + racingTotalTimeoutMs: 60_000, + stickyTimeoutCooldownMs: 300_000, + apiKey: "secret", + }, + summary: { + outcome: "success", + statusCode: 200, + durationMs: 5_000, + ttfbMs: 1_000, + attemptsPerRequest: 2, + maxActiveAttempts: 2, + rounds: 1, + providerMs: 3_000, + fallbackPromotions: 0, + cancelFailures: 0, + winnerOrigin: "normal", + winnerProviderId: 7, + winnerRound: 1, + rawErrorBody: "secret", + }, + events: [ + { + type: "attempt_started", + at: 2, + elapsedMs: 1, + attemptId: "attempt-1", + attemptKind: "normal", + provider: { + id: 7, + name: "Provider 7", + priority: 1, + endpointUrl: "secret", + }, + rawErrorBody: "secret", + apiKey: "secret", + }, + ], + }); + + expect(trace?.events).toEqual([ + { + type: "attempt_started", + at: 2, + elapsedMs: 1, + attemptId: "attempt-1", + attemptKind: "normal", + provider: { id: 7, name: "Provider 7", priority: 1 }, + }, + ]); + expect(trace?.config).not.toHaveProperty("apiKey"); + expect(trace?.summary).not.toHaveProperty("rawErrorBody"); + }); + + it("限制事件数量并标记 truncated", () => { + const events = Array.from({ length: ROUTING_TRACE_MAX_EVENTS + 1 }, (_, index) => ({ + type: "round_started", + at: index, + elapsedMs: index, + round: index + 1, + })); + + const trace = normalizeRoutingTrace({ + version: 1, + mode: "discovery", + startedAt: 0, + updatedAt: 1, + discoveryEnabled: true, + eligible: true, + events, + }); + + expect(trace?.events).toHaveLength(ROUTING_TRACE_MAX_EVENTS); + expect(trace?.truncated).toBe(true); + }); +}); diff --git a/tests/unit/usage-ledger/trigger.test.ts b/tests/unit/usage-ledger/trigger.test.ts index 33bdeb8a3..a8c8d804f 100644 --- a/tests/unit/usage-ledger/trigger.test.ts +++ b/tests/unit/usage-ledger/trigger.test.ts @@ -37,4 +37,9 @@ describe("fn_upsert_usage_ledger trigger SQL", () => { it("creates trigger binding", () => { expect(sql).toContain("CREATE TRIGGER trg_upsert_usage_ledger"); }); + + it("does not run the accounting projection for routing-trace-only updates", () => { + expect(sql).toContain("AFTER INSERT OR UPDATE OF"); + expect(sql).not.toMatch(/UPDATE OF[\s\S]*routing_trace[\s\S]*ON message_request/); + }); }); From 295b24fa190aef06938d9e0bd68c606c3c506be5 Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Tue, 21 Jul 2026 05:48:57 -0400 Subject: [PATCH 02/12] test: complete live routing trace mocks --- tests/integration/proxy-hedge-lifecycle.test.ts | 1 + tests/unit/proxy/hedge-winner-dedup.test.ts | 1 + tests/unit/proxy/response-handler-abort-listener-cleanup.test.ts | 1 + tests/unit/proxy/response-handler-client-abort-drain.test.ts | 1 + 4 files changed, 4 insertions(+) diff --git a/tests/integration/proxy-hedge-lifecycle.test.ts b/tests/integration/proxy-hedge-lifecycle.test.ts index 7707cabb6..80e3b2841 100644 --- a/tests/integration/proxy-hedge-lifecycle.test.ts +++ b/tests/integration/proxy-hedge-lifecycle.test.ts @@ -244,6 +244,7 @@ vi.mock("@/lib/proxy-status-tracker", () => ({ vi.mock("@/lib/redis/live-chain-store", () => ({ deleteLiveChain: vi.fn(async () => {}), writeLiveChain: vi.fn(async () => {}), + writeLiveRoutingTrace: vi.fn(async () => {}), })); const CREATED_AT = new Date(0); diff --git a/tests/unit/proxy/hedge-winner-dedup.test.ts b/tests/unit/proxy/hedge-winner-dedup.test.ts index ca78eebb7..3c37a7abb 100644 --- a/tests/unit/proxy/hedge-winner-dedup.test.ts +++ b/tests/unit/proxy/hedge-winner-dedup.test.ts @@ -115,6 +115,7 @@ vi.mock("@/repository/provider", () => ({ vi.mock("@/lib/redis/live-chain-store", () => ({ writeLiveChain: vi.fn(), + writeLiveRoutingTrace: vi.fn(), })); import { ProxySession } from "@/app/v1/_lib/proxy/session"; diff --git a/tests/unit/proxy/response-handler-abort-listener-cleanup.test.ts b/tests/unit/proxy/response-handler-abort-listener-cleanup.test.ts index c18814926..02d7446d1 100644 --- a/tests/unit/proxy/response-handler-abort-listener-cleanup.test.ts +++ b/tests/unit/proxy/response-handler-abort-listener-cleanup.test.ts @@ -74,6 +74,7 @@ vi.mock("@/lib/rate-limit", () => ({ vi.mock("@/lib/redis/live-chain-store", () => ({ deleteLiveChain: vi.fn(), + writeLiveRoutingTrace: vi.fn(), })); vi.mock("@/lib/session-manager", () => ({ diff --git a/tests/unit/proxy/response-handler-client-abort-drain.test.ts b/tests/unit/proxy/response-handler-client-abort-drain.test.ts index 17a125f67..a2b0096bc 100644 --- a/tests/unit/proxy/response-handler-client-abort-drain.test.ts +++ b/tests/unit/proxy/response-handler-client-abort-drain.test.ts @@ -114,6 +114,7 @@ vi.mock("@/lib/rate-limit", () => ({ vi.mock("@/lib/redis/live-chain-store", () => ({ deleteLiveChain: vi.fn(), + writeLiveRoutingTrace: vi.fn(), })); vi.mock("@/lib/session-manager", () => ({ From 0819bfb1d045a0d53d5f35a95c01bcd42e86d1c7 Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Tue, 21 Jul 2026 06:16:13 -0400 Subject: [PATCH 03/12] fix(discovery): close review gaps in trace and stream finalization --- src/app/v1/_lib/proxy/forwarder.ts | 102 ++- src/app/v1/_lib/proxy/response-handler.ts | 222 +++--- src/app/v1/_lib/proxy/stream-finalization.ts | 4 +- src/repository/message.ts | 37 +- .../proxy-forwarder-hedge-first-byte.test.ts | 706 +++++++++++++++--- ...esponse-handler-client-abort-drain.test.ts | 56 +- ...handler-endpoint-circuit-isolation.test.ts | 243 +++++- .../message-terminal-write-apis.test.ts | 159 ++++ 8 files changed, 1239 insertions(+), 290 deletions(-) diff --git a/src/app/v1/_lib/proxy/forwarder.ts b/src/app/v1/_lib/proxy/forwarder.ts index d3f971ba3..21f857e35 100644 --- a/src/app/v1/_lib/proxy/forwarder.ts +++ b/src/app/v1/_lib/proxy/forwarder.ts @@ -467,14 +467,18 @@ async function readResponseTextUpTo( // because the tee controller waits for internal queue drainage while the other // branch has not started consuming yet. This deadlocks the main request path. reader.cancel().catch((cancelErr) => { - logger.debug("readResponseTextUpTo: failed to cancel reader", { error: cancelErr }); + logger.debug("readResponseTextUpTo: failed to cancel reader", { + error: cancelErr, + }); }); } try { reader.releaseLock(); } catch (releaseErr) { - logger.debug("readResponseTextUpTo: failed to release reader lock", { error: releaseErr }); + logger.debug("readResponseTextUpTo: failed to release reader lock", { + error: releaseErr, + }); } } @@ -1412,10 +1416,16 @@ export class ProxyForwarder { providerVendorId > 0; let endpointSelectionError: Error | null = null; - const endpointCandidates: Array<{ endpointId: number | null; baseUrl: string }> = []; + const endpointCandidates: Array<{ + endpointId: number | null; + baseUrl: string; + }> = []; if (isMcpRequest) { - endpointCandidates.push({ endpointId: null, baseUrl: currentProvider.url }); + endpointCandidates.push({ + endpointId: null, + baseUrl: currentProvider.url, + }); } else if (providerVendorId > 0) { try { const preferred = await getPreferredProviderEndpoints({ @@ -1459,7 +1469,9 @@ export class ProxyForwarder { ); // Record endpoint pool exhaustion in provider chain for audit trail - const exhaustionContext: Record = { strictBlockCause }; + const exhaustionContext: Record = { + strictBlockCause, + }; if (endpointSelectionError) { exhaustionContext.selectorError = endpointSelectionError.message; } @@ -1513,7 +1525,10 @@ export class ProxyForwarder { ProxyForwarder.markProviderFailed(session, failedProviderIds, currentProvider.id); attemptCount = maxAttemptsPerProvider; } else { - endpointCandidates.push({ endpointId: null, baseUrl: currentProvider.url }); + endpointCandidates.push({ + endpointId: null, + baseUrl: currentProvider.url, + }); } } @@ -3593,7 +3608,10 @@ export class ProxyForwarder { // 记录到决策链（标记为 HTTP/2 回退） session.addProviderToChain(provider, { - ...(endpointAudit ?? { endpointId: null, endpointUrl: sanitizeUrl(baseUrl) }), + ...(endpointAudit ?? { + endpointId: null, + endpointUrl: sanitizeUrl(baseUrl), + }), reason: "http2_fallback", circuitState: getCircuitState(provider.id), attemptNumber: attemptNumber ?? 1, @@ -5171,7 +5189,10 @@ export class ProxyForwarder { const roundLaunchIdleWaiters = new Set<() => void>(); const queuedRoundLaunchStartWaiters = new Set<() => void>(); let fallbackPromotionBlocked = false; - type StickyTimeoutWaveReservation = { fallbackAttemptId: string; slots: number }; + type StickyTimeoutWaveReservation = { + fallbackAttemptId: string; + slots: number; + }; let stickyTimeoutWaveReservation: StickyTimeoutWaveReservation | null = null; let stickyTimeoutWaveClaim: StickyTimeoutWaveReservation | null = null; let stickyTimeoutWaveLaunchPromise: Promise | null = null; @@ -5393,7 +5414,10 @@ export class ProxyForwarder { ); } catch (error) { discoveryMetrics.cancelFailed(attempt.id, attempt.provider.id, error); - logger.debug("[Discovery] Reader cancel failed", { cancellationKind, error }); + logger.debug("[Discovery] Reader cancel failed", { + cancellationKind, + error, + }); } } if (attempt.releaseAgent && !attempt.agentReleased) { @@ -5617,6 +5641,12 @@ export class ProxyForwarder { if (attempt.session !== session) ProxyForwarder.syncWinningAttemptSession(session, attempt.session); + const bindingIntent = + attempt.kind === "fallback" || !bindingWriteAllowed || !session.isSessionBindingAllowed() + ? "none" + : bindingSnapshot?.providerId == null + ? "create" + : "renew"; setDeferredStreamingFinalization(session, { providerId: attempt.provider.id, providerName: attempt.provider.name, @@ -5630,14 +5660,9 @@ export class ProxyForwarder { upstreamStatusCode: attempt.response.status, isHedgeWinner: false, billHedgeLosers: false, - bindingIntent: - attempt.kind === "fallback" || !bindingWriteAllowed || !session.isSessionBindingAllowed() - ? "none" - : bindingSnapshot?.providerId == null - ? "create" - : "renew", + bindingIntent, bindingSnapshot, - requiresCompletionMarker: true, + requiresCompletionMarkerForBinding: bindingIntent === "create" || bindingIntent === "renew", discoveryLease: lease, providerSessionRefOwned: attempt.providerSessionRefOwned, providerSessionRefRetainOnSuccess: attempt.providerSessionRefRetainOnSuccess, @@ -5945,7 +5970,10 @@ export class ProxyForwarder { provider, session: attemptSession, baseUrl: endpoint.baseUrl, - endpointAudit: { endpointId: endpoint.endpointId, endpointUrl: endpoint.endpointUrl }, + endpointAudit: { + endpointId: endpoint.endpointId, + endpointUrl: endpoint.endpointUrl, + }, modelRedirect: undefined, responseController: null, clearResponseTimeout: null, @@ -6876,10 +6904,6 @@ export class ProxyForwarder { if (candidateSetupReservation) { cancelSetupReservation(candidateSetupReservation, "discovery_sla_timeout"); } - // Coordinator marks cancelled attempts non-pending before returning - // the action. Restore the transport-facing state long enough for the - // exactly-once cancellation/release path to run. - if (attempt && !attempt.readerTransferred) attempt.pending = true; if (attempt) cancelAttempt(attempt, "discovery_sla_timeout"); } if ("promoteAttemptId" in action && action.promoteAttemptId) { @@ -7022,13 +7046,17 @@ export class ProxyForwarder { Math.ceil((settings.stickyTimeoutCooldownMs ?? 300_000) / 1000) ).finally(() => { void launchReservedStickyTimeoutWave().catch((error) => - logger.warn("[Discovery] Sticky round launch failed", { error }) + logger.warn("[Discovery] Sticky round launch failed", { + error, + }) ); }); return; } void launchReservedStickyTimeoutWave().catch((error) => - logger.warn("[Discovery] Sticky round launch failed", { error }) + logger.warn("[Discovery] Sticky round launch failed", { + error, + }) ); } }, @@ -7086,7 +7114,11 @@ export class ProxyForwarder { private static async resolveStreamingHedgeEndpoint( session: ProxySession, provider: Provider - ): Promise<{ endpointId: number | null; baseUrl: string; endpointUrl: string }> { + ): Promise<{ + endpointId: number | null; + baseUrl: string; + endpointUrl: string; + }> { const requestPath = session.requestUrl.pathname; const providerVendorId = provider.providerVendorId ?? 0; const isMcpRequest = @@ -7110,13 +7142,20 @@ export class ProxyForwarder { }); } - const endpointCandidates: Array<{ endpointId: number | null; endpointUrl: string }> = []; + const endpointCandidates: Array<{ + endpointId: number | null; + endpointUrl: string; + }> = []; let endpointSelectionError: Error | null = null; if (isMcpRequest) { const sanitizedUrl = sanitizeUrl(provider.url); endpointCandidates.push({ endpointId: null, endpointUrl: sanitizedUrl }); - return { endpointId: null, baseUrl: provider.url, endpointUrl: sanitizedUrl }; + return { + endpointId: null, + baseUrl: provider.url, + endpointUrl: sanitizedUrl, + }; } if (providerVendorId > 0) { @@ -7126,7 +7165,10 @@ export class ProxyForwarder { providerType: provider.providerType, }); endpointCandidates.push( - ...preferred.map((endpoint) => ({ endpointId: endpoint.id, endpointUrl: endpoint.url })) + ...preferred.map((endpoint) => ({ + endpointId: endpoint.id, + endpointUrl: endpoint.url, + })) ); } catch (error) { endpointSelectionError = error instanceof Error ? error : new Error(String(error)); @@ -7159,7 +7201,11 @@ export class ProxyForwarder { } const sanitizedUrl = sanitizeUrl(provider.url); - return { endpointId: null, baseUrl: provider.url, endpointUrl: sanitizedUrl }; + return { + endpointId: null, + baseUrl: provider.url, + endpointUrl: sanitizedUrl, + }; } return { diff --git a/src/app/v1/_lib/proxy/response-handler.ts b/src/app/v1/_lib/proxy/response-handler.ts index 9d6f96f88..855bd7ef4 100644 --- a/src/app/v1/_lib/proxy/response-handler.ts +++ b/src/app/v1/_lib/proxy/response-handler.ts @@ -50,7 +50,7 @@ import type { SessionUsageUpdate } from "@/types/session"; import type { LongContextPricingSpecialSetting } from "@/types/special-settings"; import { GeminiAdapter } from "../gemini/adapter"; import type { GeminiResponse } from "../gemini/types"; -import { extractActualResponseModelForProvider } from "./actual-response-model"; +import { extractActualResponseModelForProvider, extractJsonChunks } from "./actual-response-model"; import { bindClientAbortListener } from "./client-abort-listener"; import { createDemandDrivenResponsePump, @@ -880,7 +880,9 @@ function bindTaskAbortToUpstreamResponse( } }; - abortController.signal.addEventListener("abort", abortUpstream, { once: true }); + abortController.signal.addEventListener("abort", abortUpstream, { + once: true, + }); if (abortController.signal.aborted) { abortUpstream(); } @@ -1303,41 +1305,73 @@ function hasGeminiCompletionMarker(data: unknown): boolean { * 仅 usage>0 不足以证明完成：Anthropic 在首个 `message_start` 即带 usage、 * Gemini 在中间事件即带 usageMetadata，截断流同样会出现正向 token。 */ -function hasStreamCompletionMarker(text: string, format: ProxySession["originalFormat"]): boolean { +function inspectStreamCompletion( + text: string, + format: ProxySession["originalFormat"] +): { hasMarker: boolean; hasProtocolError: boolean } { const events = parseSSEData(text); + const payloads: unknown[] = events.map((event) => event.data); + + // Native Gemini passthrough is NDJSON rather than SSE. Reuse the shared + // structured stream extractor so a valid finishReason can authorize Sticky. + if (format === "gemini" || format === "gemini-cli") { + for (const candidate of extractJsonChunks(text)) { + try { + payloads.push(JSON.parse(candidate) as unknown); + } catch { + // Malformed or incomplete JSON cannot establish completion. + } + } + } + + const hasProtocolError = payloads.some(isDiscoveryProtocolErrorPayload); + if (hasProtocolError) return { hasMarker: false, hasProtocolError: true }; switch (format) { case "response": - if (events.some((event) => isDiscoveryProtocolErrorPayload(event.data))) return false; - return events.some((event) => { - if (!isRecord(event.data)) return false; - const markerType = event.data.type; - if (markerType !== "response.completed" && markerType !== "response.done") return false; - if (event.event !== "message" && event.event !== markerType) return false; - return markerType === "response.done" || isRecord(event.data.response); - }); + return { + hasMarker: events.some((event) => { + if (!isRecord(event.data)) return false; + const markerType = event.data.type; + if (markerType !== "response.completed" && markerType !== "response.done") return false; + if (event.event !== "message" && event.event !== markerType) return false; + return markerType === "response.done" || isRecord(event.data.response); + }), + hasProtocolError: false, + }; case "claude": - return events.some( - (event) => - (event.event === "message_stop" || event.event === "message") && - isRecord(event.data) && - event.data.type === "message_stop" - ); + return { + hasMarker: events.some( + (event) => + (event.event === "message_stop" || event.event === "message") && + isRecord(event.data) && + event.data.type === "message_stop" + ), + hasProtocolError: false, + }; case "openai": - return events.some( - (event) => - event.event === "message" && - ((typeof event.data === "string" && event.data.trim() === "[DONE]") || - hasOpenAIChatCompletionMarker(event.data)) - ); + return { + hasMarker: events.some( + (event) => + event.event === "message" && + ((typeof event.data === "string" && event.data.trim() === "[DONE]") || + hasOpenAIChatCompletionMarker(event.data)) + ), + hasProtocolError: false, + }; case "gemini": case "gemini-cli": - return events.some( - (event) => event.event === "message" && hasGeminiCompletionMarker(event.data) - ); + return { + hasMarker: payloads.some(hasGeminiCompletionMarker), + hasProtocolError: false, + }; } } +function hasStreamCompletionMarker(text: string, format: ProxySession["originalFormat"]): boolean { + return inspectStreamCompletion(text, format).hasMarker; +} + export async function resolveBillableUsageMetricsForCost( session: ProxySession, provider: Provider | null, @@ -1523,7 +1557,7 @@ function finalizeDeferredStreamingFinalizationIfNeeded( await updateMessageRequestRoutingTrace(session.messageContext.id, routingTrace); } }; - const clearSessionBinding = async () => { + const clearSessionBinding = async (bindingFailureReason?: string) => { if (!session.sessionId || !isSessionBindingMutationAllowed(session)) return; const hedgeAuthority = meta?.isHedgeWinner ? await meta.hedgeBindingAuthorityPromise @@ -1551,7 +1585,10 @@ function finalizeDeferredStreamingFinalizationIfNeeded( await recordBindingFinalized({ bindingAction: "none", outcome: "skipped", - reason: meta.bindingIntent === "create" ? "stream_not_successful" : "binding_not_requested", + reason: + meta.bindingIntent === "create" + ? (bindingFailureReason ?? "stream_not_successful") + : "binding_not_requested", }); return; } @@ -1612,7 +1649,8 @@ function finalizeDeferredStreamingFinalizationIfNeeded( await recordBindingFinalized({ bindingAction: "clear", outcome: cleared.status === "ok" ? "cleared" : "skipped", - reason: cleared.status === "ok" ? "stream_failed" : cleared.reason, + reason: + cleared.status === "ok" ? (bindingFailureReason ?? "stream_failed") : cleared.reason, }); return; } @@ -1636,7 +1674,11 @@ function finalizeDeferredStreamingFinalizationIfNeeded( snapshot.keyId !== keyId || (session.sessionId !== null && snapshot.sessionId !== session.sessionId) ) - return { updated: false, reason: "missing_snapshot", details: "missing_snapshot" }; + return { + updated: false, + reason: "missing_snapshot", + details: "missing_snapshot", + }; if (!(await discoveryLeaseLifecycle.ensureOwned())) { return { @@ -1661,7 +1703,11 @@ function finalizeDeferredStreamingFinalizationIfNeeded( outcome: "failed", reason: "binding_error", }); - return { updated: false, reason: "binding_error", details: "binding_error" }; + return { + updated: false, + reason: "binding_error", + details: "binding_error", + }; } if (cas.status === "conflict") { recordDiscoveryControlEvent("binding_cas_conflict", { @@ -1688,11 +1734,18 @@ function finalizeDeferredStreamingFinalizationIfNeeded( const isHedgeWinner = meta?.isHedgeWinner === true; const billHedgeLosers = meta?.billHedgeLosers === true; + const hasDiscoveryBindingIntent = + meta?.bindingIntent === "create" || meta?.bindingIntent === "renew"; + const completionInspection = inspectStreamCompletion(allContent, session.originalFormat); + const completionMarkerMissingForBinding = + meta?.requiresCompletionMarkerForBinding === true && + hasDiscoveryBindingIntent && + streamEndedNormally && + !completionInspection.hasMarker; const allowAuxiliarySessionBinding = isSessionBindingMutationAllowed(session) && + !completionMarkerMissingForBinding && (meta?.bindingIntent === undefined || (meta.bindingIntent !== "none" && !clientAborted)); - const hasDiscoveryBindingIntent = - meta?.bindingIntent === "create" || meta?.bindingIntent === "renew"; let resolvePrimaryDiscoveryBinding: ((updated: boolean) => void) | null = null; const primaryDiscoveryBinding = hasDiscoveryBindingIntent ? new Promise((resolve) => { @@ -1723,13 +1776,17 @@ function finalizeDeferredStreamingFinalizationIfNeeded( // // 此处返回 `{isError:false}` 仅表示“跳过检测”，最终仍会在下面按中断/超时视为失败结算。 const shouldDetectFake200 = streamEndedNormally && upstreamStatusCode === 200; - const detected = shouldDetectFake200 + const bodyDetected = shouldDetectFake200 ? detectUpstreamErrorFromSseOrJsonText(allContent) : ({ isError: false } as const); - const completionMarkerMissing = - meta?.requiresCompletionMarker === true && - streamEndedNormally && - !hasStreamCompletionMarker(allContent, session.originalFormat); + const detected = + shouldDetectFake200 && !bodyDetected.isError && completionInspection.hasProtocolError + ? ({ + isError: true, + code: "UPSTREAM_PROTOCOL_ERROR", + detail: "Upstream stream emitted a protocol error event", + } as const) + : bodyDetected; let clientAbortGateUsage: FinalizeDeferredStreamingResult["clientAbortGateUsage"]; const clientAbortCompleteSuccess = (() => { if (!clientAborted || upstreamStatusCode < 200 || upstreamStatusCode >= 300) { @@ -1753,7 +1810,10 @@ function finalizeDeferredStreamingFinalizationIfNeeded( } const { usageMetrics } = parseUsageFromResponseText(allContent, provider?.providerType); - clientAbortGateUsage = { usageMetrics, providerType: provider?.providerType }; + clientAbortGateUsage = { + usageMetrics, + providerType: provider?.providerType, + }; return hasPositiveBillableTokens(usageMetrics); })(); @@ -1774,9 +1834,6 @@ function finalizeDeferredStreamingFinalizationIfNeeded( effectiveStatusCode = 502; } errorMessage = detected.detail ? `${detected.code}: ${detected.detail}` : detected.code; - } else if (completionMarkerMissing) { - effectiveStatusCode = 502; - errorMessage = "STREAM_COMPLETION_MARKER_MISSING"; } else if (clientAbortCompleteSuccess) { effectiveStatusCode = upstreamStatusCode; errorMessage = null; @@ -1807,7 +1864,6 @@ function finalizeDeferredStreamingFinalizationIfNeeded( const shouldClearSessionBindingOnFailure = ((clientAborted || !streamEndedNormally) && !clientAbortCompleteSuccess) || detected.isError || - completionMarkerMissing || (upstreamStatusCode >= 400 && errorMessage !== null); if (shouldClearSessionBindingOnFailure) { meta?.hedgeBindingHeartbeat?.stop(); @@ -1921,50 +1977,6 @@ function finalizeDeferredStreamingFinalizationIfNeeded( }; } - if (completionMarkerMissing) { - session.addProviderToChain(providerForChain, { - endpointId: meta.endpointId, - endpointUrl: meta.endpointUrl, - reason: "retry_failed", - attemptNumber: meta.attemptNumber, - statusCode: effectiveStatusCode, - errorMessage: errorMessage ?? undefined, - }); - - const commitSideEffects = async () => { - try { - await clearSessionBinding(); - if (session.getEndpointPolicy().allowCircuitBreakerAccounting) { - try { - const { recordFailure } = await import("@/lib/circuit-breaker"); - await recordFailure(meta.providerId, new Error(errorMessage ?? "STREAM_ABORTED")); - } catch (cbError) { - logger.warn("[ResponseHandler] Failed to record missing stream completion marker", { - providerId: meta.providerId, - sessionId: session.sessionId ?? null, - error: cbError, - }); - } - } - } finally { - await finalizeFailedDiscoveryBinding(); - } - }; - - return { - effectiveStatusCode, - errorMessage, - providerIdForPersistence, - isHedgeWinner, - billHedgeLosers, - clientAbortGateUsage, - commitSideEffects, - finalizeAttemptResources: finalizeProviderSessionRef, - allowAuxiliarySessionBinding, - confirmAuxiliarySessionBinding, - }; - } - if (detected.isError) { logger.warn("[ResponseHandler] SSE completed but body indicates error (fake 200)", { providerId: meta.providerId, @@ -2146,9 +2158,19 @@ function finalizeDeferredStreamingFinalizationIfNeeded( }); } - // A client abort may still be billable when a completion marker was - // already buffered, but it must never create or renew Sticky state. - if ( + // A natural 2xx EOF without a protocol marker remains a successful, + // billable request, but it cannot create or renew Sticky state. + if (completionMarkerMissingForBinding) { + if (meta.bindingIntent === "renew") { + await clearSessionBinding("completion_marker_missing"); + } else if (meta.bindingIntent === "create") { + await recordBindingFinalized({ + bindingAction: "create", + outcome: "skipped", + reason: "completion_marker_missing", + }); + } + } else if ( meta.bindingIntent !== "none" && !meta.isHedgeWinner && !clientAborted && @@ -2161,7 +2183,11 @@ function finalizeDeferredStreamingFinalizationIfNeeded( const result = isDiscoveryBinding ? meta.bindingSnapshot && keyId != null ? await compareAndSetDiscoveryBinding(meta.bindingSnapshot, meta.providerId, keyId) - : { updated: false, reason: "missing_snapshot", details: "missing_snapshot" } + : { + updated: false, + reason: "missing_snapshot", + details: "missing_snapshot", + } : await SessionManager.updateSessionBindingSmart( session.sessionId, meta.providerId, @@ -6170,11 +6196,17 @@ async function trackCostToRedis( }, user: { id: user.id, - resetModes: { "5h": user.limit5hResetMode, daily: user.dailyResetMode }, + resetModes: { + "5h": user.limit5hResetMode, + daily: user.dailyResetMode, + }, }, provider: { id: provider.id, - resetModes: { "5h": provider.limit5hResetMode, daily: provider.dailyResetMode }, + resetModes: { + "5h": provider.limit5hResetMode, + daily: provider.dailyResetMode, + }, }, }, }); @@ -6281,7 +6313,9 @@ async function persistRequestFailure(options: { specialSettings: session.getSpecialSettings() ?? undefined, }; const persistence = options.onCommitted - ? detailsWriter(messageContext.id, terminalDetails, { onCommitted: options.onCommitted }) + ? detailsWriter(messageContext.id, terminalDetails, { + onCommitted: options.onCommitted, + }) : detailsWriter(messageContext.id, terminalDetails); committed = Boolean(await awaitPersistence(persistence)); diff --git a/src/app/v1/_lib/proxy/stream-finalization.ts b/src/app/v1/_lib/proxy/stream-finalization.ts index 1184f2afd..383266c0d 100644 --- a/src/app/v1/_lib/proxy/stream-finalization.ts +++ b/src/app/v1/_lib/proxy/stream-finalization.ts @@ -58,8 +58,8 @@ export type DeferredStreamingFinalization = { /** Discovery delays binding until the stream has a valid completion marker. */ bindingIntent?: "create" | "renew" | "none"; bindingSnapshot?: SessionBindingSnapshot | null; - /** Discovery winners must satisfy the protocol completion marker before binding. */ - requiresCompletionMarker?: boolean; + /** Discovery create/renew intents must satisfy the protocol completion marker before binding. */ + requiresCompletionMarkerForBinding?: boolean; /** Lease already acquired by Forwarder and owned until terminal side effects finish. */ discoveryLease?: DeferredStreamingDiscoveryLease; /** Whether this attempt owns a Provider concurrent-session reference. */ diff --git a/src/repository/message.ts b/src/repository/message.ts index c12167f7c..e4c187ef1 100644 --- a/src/repository/message.ts +++ b/src/repository/message.ts @@ -626,22 +626,31 @@ export async function updateMessageRequestRoutingTrace( const normalized = normalizeRoutingTrace(routingTrace); if (!normalized) return; - if (getEnvConfig().MESSAGE_REQUEST_WRITE_MODE === "async") { - enqueueMessageRequestUpdate(id, { routingTrace: normalized }); - return; - } + const useWriterLane = getEnvConfig().MESSAGE_REQUEST_WRITE_MODE === "async"; + const maxAttempts = 3; + let lastError: unknown; - try { - await db - .update(messageRequest) - .set({ routingTrace: normalized, updatedAt: new Date() }) - .where(and(eq(messageRequest.id, id), isNull(messageRequest.deletedAt))); - } catch (error) { - logger.warn("[MessageRequest] Failed to patch finalized routing trace", { - requestId: id, - error: error instanceof Error ? error.message : String(error), - }); + for (let attempt = 0; attempt < maxAttempts; attempt++) { + try { + const traceDb = useWriterLane ? getMessageWriterDb() : db; + await traceDb + .update(messageRequest) + .set({ routingTrace: normalized, updatedAt: new Date() }) + .where(and(eq(messageRequest.id, id), isNull(messageRequest.deletedAt))); + return; + } catch (error) { + lastError = error; + if (attempt < maxAttempts - 1) { + await new Promise((resolve) => setTimeout(resolve, 50 * (attempt + 1))); + } + } } + + logger.warn("[MessageRequest] Failed to patch finalized routing trace", { + requestId: id, + attempts: maxAttempts, + error: lastError instanceof Error ? lastError.message : String(lastError), + }); } export async function updateMessageRequestDetailsIfUnfinalized( diff --git a/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts b/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts index 76ddca059..ce0a09001 100644 --- a/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts +++ b/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts @@ -14,7 +14,10 @@ const mocks = vi.hoisted(() => ({ health: { failureCount: 0 }, config: { failureThreshold: 3 }, })), - updateSessionBindingSmart: vi.fn(async () => ({ updated: true, reason: "test" })), + updateSessionBindingSmart: vi.fn(async () => ({ + updated: true, + reason: "test", + })), updateSessionProvider: vi.fn(async () => {}), clearSessionProvider: vi.fn(async () => {}), clearSessionProviders: vi.fn(async () => false), @@ -98,7 +101,10 @@ vi.mock("@/lib/provider-endpoints/endpoint-selector", () => ({ vi.mock("@/app/v1/_lib/responses-ws/eligibility", async (importOriginal) => { const actual = await importOriginal(); - return { ...actual, isWebsocketClientRequest: mocks.isWebsocketClientRequest }; + return { + ...actual, + isWebsocketClientRequest: mocks.isWebsocketClientRequest, + }; }); vi.mock("@/lib/endpoint-circuit-breaker", () => ({ @@ -317,7 +323,9 @@ function createStreamingResponse(params: { return; } - params.controller.signal.addEventListener("abort", onAbort, { once: true }); + params.controller.signal.addEventListener("abort", onAbort, { + once: true, + }); timeoutId = setTimeout(() => { if (params.controller.signal.aborted) { controller.close(); @@ -355,7 +363,9 @@ function createDelayedFailure(params: { return; } - params.controller.signal.addEventListener("abort", rejectWithError, { once: true }); + params.controller.signal.addEventListener("abort", rejectWithError, { + once: true, + }); timeoutId = setTimeout(() => { params.controller.signal.removeEventListener("abort", rejectWithError); reject(params.error); @@ -522,7 +532,13 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { const fireworks = createProvider({ id: 383, name: "fireworks", - modelRedirects: [{ matchType: "exact", source: requestedModel, target: fireworksRedirect }], + modelRedirects: [ + { + matchType: "exact", + source: requestedModel, + target: fireworksRedirect, + }, + ], }); const minimax = createProvider({ id: 206, @@ -591,7 +607,13 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { const fireworks = createProvider({ id: 383, name: "fireworks", - modelRedirects: [{ matchType: "exact", source: requestedModel, target: fireworksRedirect }], + modelRedirects: [ + { + matchType: "exact", + source: requestedModel, + target: fireworksRedirect, + }, + ], }); const fallback = createProvider({ id: 206, @@ -649,7 +671,13 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { const fireworks = createProvider({ id: 383, name: "fireworks", - modelRedirects: [{ matchType: "exact", source: requestedModel, target: fireworksRedirect }], + modelRedirects: [ + { + matchType: "exact", + source: requestedModel, + target: fireworksRedirect, + }, + ], }); const plainProvider = createProvider({ id: 520, @@ -697,13 +725,25 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { id: 383, name: "fireworks", firstByteTimeoutStreamingMs: 100, - modelRedirects: [{ matchType: "exact", source: requestedModel, target: fireworksRedirect }], + modelRedirects: [ + { + matchType: "exact", + source: requestedModel, + target: fireworksRedirect, + }, + ], }); const minimax = createProvider({ id: 206, name: "Minimax Max", firstByteTimeoutStreamingMs: 100, - modelRedirects: [{ matchType: "exact", source: requestedModel, target: minimaxRedirect }], + modelRedirects: [ + { + matchType: "exact", + source: requestedModel, + target: minimaxRedirect, + }, + ], }); const session = createSession(); session.request.model = requestedModel; @@ -793,8 +833,16 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { vi.useFakeTimers(); try { - const slow = createProvider({ id: 383, name: "slow", firstByteTimeoutStreamingMs: 100 }); - const fast = createProvider({ id: 206, name: "fast", firstByteTimeoutStreamingMs: 100 }); + const slow = createProvider({ + id: 383, + name: "slow", + firstByteTimeoutStreamingMs: 100, + }); + const fast = createProvider({ + id: 206, + name: "fast", + firstByteTimeoutStreamingMs: 100, + }); const session = createSession(); setProviderWithSessionRef(session, slow); session.addProviderToChain(slow, { reason: "initial_selection" }); @@ -869,13 +917,25 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { id: 383, name: "fireworks", firstByteTimeoutStreamingMs: 100, - modelRedirects: [{ matchType: "exact", source: requestedModel, target: fireworksRedirect }], + modelRedirects: [ + { + matchType: "exact", + source: requestedModel, + target: fireworksRedirect, + }, + ], }); const minimax = createProvider({ id: 206, name: "Minimax Max", firstByteTimeoutStreamingMs: 100, - modelRedirects: [{ matchType: "exact", source: requestedModel, target: minimaxRedirect }], + modelRedirects: [ + { + matchType: "exact", + source: requestedModel, + target: minimaxRedirect, + }, + ], }); const session = createSession(); session.request.model = requestedModel; @@ -1092,8 +1152,16 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { vi.useFakeTimers(); try { - const provider1 = createProvider({ id: 1, name: "p1", firstByteTimeoutStreamingMs: 100 }); - const provider2 = createProvider({ id: 2, name: "p2", firstByteTimeoutStreamingMs: 100 }); + const provider1 = createProvider({ + id: 1, + name: "p1", + firstByteTimeoutStreamingMs: 100, + }); + const provider2 = createProvider({ + id: 2, + name: "p2", + firstByteTimeoutStreamingMs: 100, + }); const session = createSession(); session.authState = { ...session.authState!, @@ -1185,14 +1253,22 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { vi.useFakeTimers(); try { - const provider1 = createProvider({ id: 1, name: "p1", firstByteTimeoutStreamingMs: 100 }); + const provider1 = createProvider({ + id: 1, + name: "p1", + firstByteTimeoutStreamingMs: 100, + }); const provider2 = createProvider({ id: 2, name: "p2", firstByteTimeoutStreamingMs: 100, limitConcurrentSessions: 1, }); - const provider3 = createProvider({ id: 3, name: "p3", firstByteTimeoutStreamingMs: 100 }); + const provider3 = createProvider({ + id: 3, + name: "p3", + firstByteTimeoutStreamingMs: 100, + }); const session = createSession(); setProviderWithSessionRef(session, provider1); @@ -1207,7 +1283,12 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { referenced: false, reason: "供应商并发 Session 上限已达到（1/1）", }) - .mockResolvedValueOnce({ allowed: true, count: 1, tracked: true, referenced: true }); + .mockResolvedValueOnce({ + allowed: true, + count: 1, + tracked: true, + referenced: true, + }); const doForward = vi.spyOn( ProxyForwarder as unknown as { @@ -1278,8 +1359,16 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { vi.useFakeTimers(); try { - const provider1 = createProvider({ id: 1, name: "p1", firstByteTimeoutStreamingMs: 100 }); - const provider2 = createProvider({ id: 2, name: "p2", firstByteTimeoutStreamingMs: 100 }); + const provider1 = createProvider({ + id: 1, + name: "p1", + firstByteTimeoutStreamingMs: 100, + }); + const provider2 = createProvider({ + id: 2, + name: "p2", + firstByteTimeoutStreamingMs: 100, + }); const session = createSession(); session.setHighConcurrencyModeEnabled(true); setProviderWithSessionRef(session, provider1); @@ -1414,8 +1503,16 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { vi.useFakeTimers(); try { - const provider1 = createProvider({ id: 1, name: "p1", firstByteTimeoutStreamingMs: 100 }); - const provider2 = createProvider({ id: 2, name: "p2", firstByteTimeoutStreamingMs: 100 }); + const provider1 = createProvider({ + id: 1, + name: "p1", + firstByteTimeoutStreamingMs: 100, + }); + const provider2 = createProvider({ + id: 2, + name: "p2", + firstByteTimeoutStreamingMs: 100, + }); const session = createSession(); setProviderWithSessionRef(session, provider1); @@ -1488,9 +1585,21 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { vi.useFakeTimers(); try { - const provider1 = createProvider({ id: 1, name: "p1", firstByteTimeoutStreamingMs: 100 }); - const provider2 = createProvider({ id: 2, name: "p2", firstByteTimeoutStreamingMs: 100 }); - const provider3 = createProvider({ id: 3, name: "p3", firstByteTimeoutStreamingMs: 100 }); + const provider1 = createProvider({ + id: 1, + name: "p1", + firstByteTimeoutStreamingMs: 100, + }); + const provider2 = createProvider({ + id: 2, + name: "p2", + firstByteTimeoutStreamingMs: 100, + }); + const provider3 = createProvider({ + id: 3, + name: "p3", + firstByteTimeoutStreamingMs: 100, + }); const session = createSession(); setProviderWithSessionRef(session, provider1); @@ -1586,7 +1695,11 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { name: "p2", firstByteTimeoutStreamingMs: 100, modelRedirects: [ - { matchType: "exact", source: requestedModel, target: "MiniMax-M2.7-highspeed" }, + { + matchType: "exact", + source: requestedModel, + target: "MiniMax-M2.7-highspeed", + }, ], }); const clientAbortController = new AbortController(); @@ -1679,7 +1792,11 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { vi.useFakeTimers(); try { - const provider1 = createProvider({ id: 1, name: "p1", firstByteTimeoutStreamingMs: 100 }); + const provider1 = createProvider({ + id: 1, + name: "p1", + firstByteTimeoutStreamingMs: 100, + }); const session = createSession(); session.setProvider(provider1); @@ -1785,8 +1902,16 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { vi.useFakeTimers(); try { - const provider1 = createProvider({ id: 1, name: "p1", firstByteTimeoutStreamingMs: 100 }); - const provider2 = createProvider({ id: 2, name: "p2", firstByteTimeoutStreamingMs: 100 }); + const provider1 = createProvider({ + id: 1, + name: "p1", + firstByteTimeoutStreamingMs: 100, + }); + const provider2 = createProvider({ + id: 2, + name: "p2", + firstByteTimeoutStreamingMs: 100, + }); const session = createSession(); session.setProvider(provider1); @@ -1853,8 +1978,16 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { ); test("non-retryable client errors should stop hedge immediately and preserve original error", async () => { - const provider1 = createProvider({ id: 1, name: "p1", firstByteTimeoutStreamingMs: 100 }); - const provider2 = createProvider({ id: 2, name: "p2", firstByteTimeoutStreamingMs: 100 }); + const provider1 = createProvider({ + id: 1, + name: "p1", + firstByteTimeoutStreamingMs: 100, + }); + const provider2 = createProvider({ + id: 2, + name: "p2", + firstByteTimeoutStreamingMs: 100, + }); const session = createSession(); session.setProvider(provider1); @@ -1922,7 +2055,11 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { }); test("local DB admission overload should stop hedge without circuit mutation or failover", async () => { - const provider = createProvider({ id: 1, name: "p1", firstByteTimeoutStreamingMs: 100 }); + const provider = createProvider({ + id: 1, + name: "p1", + firstByteTimeoutStreamingMs: 100, + }); const session = createSession(); session.setProvider(provider); @@ -1968,8 +2105,16 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { vi.useFakeTimers(); try { - const provider1 = createProvider({ id: 1, name: "p1", firstByteTimeoutStreamingMs: 100 }); - const provider2 = createProvider({ id: 2, name: "p2", firstByteTimeoutStreamingMs: 100 }); + const provider1 = createProvider({ + id: 1, + name: "p1", + firstByteTimeoutStreamingMs: 100, + }); + const provider2 = createProvider({ + id: 2, + name: "p2", + firstByteTimeoutStreamingMs: 100, + }); const session = createSession(); session.setProvider(provider1); withThinkingBlocks(session); @@ -2077,8 +2222,16 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { vi.useFakeTimers(); try { - const provider1 = createProvider({ id: 1, name: "p1", firstByteTimeoutStreamingMs: 100 }); - const provider2 = createProvider({ id: 2, name: "p2", firstByteTimeoutStreamingMs: 100 }); + const provider1 = createProvider({ + id: 1, + name: "p1", + firstByteTimeoutStreamingMs: 100, + }); + const provider2 = createProvider({ + id: 2, + name: "p2", + firstByteTimeoutStreamingMs: 100, + }); const session = createSession(); session.setProvider(provider1); session.request.message = { @@ -2344,7 +2497,10 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { }); test("Discovery lease conflict forces a single upstream and forbids binding writes", async () => { - const provider = createProvider({ id: 1, firstByteTimeoutStreamingMs: 100 }); + const provider = createProvider({ + id: 1, + firstByteTimeoutStreamingMs: 100, + }); const session = createSession(); session.authState = { success: true, @@ -2387,7 +2543,10 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { }); test("foreign binding state uses single-upstream routing with serial fallback", async () => { - const provider = createProvider({ id: 1, firstByteTimeoutStreamingMs: 100 }); + const provider = createProvider({ + id: 1, + firstByteTimeoutStreamingMs: 100, + }); const alternative = createProvider({ id: 2, name: "serial-fallback" }); const session = createSession(); session.authState = { @@ -2559,7 +2718,10 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { } as typeof session.authState; session.request.message.messages = [ { role: "user", content: "first" }, - { role: "assistant", content: [{ type: "thinking", thinking: "t", signature: "sig" }] }, + { + role: "assistant", + content: [{ type: "thinking", thinking: "t", signature: "sig" }], + }, ]; setProviderWithSessionRef(session, sticky); session.setSessionBindingSnapshot({ @@ -2592,7 +2754,11 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { resolveStreamingHedgeEndpoint: ( session: ProxySession, provider: Provider - ) => Promise<{ endpointId: number | null; baseUrl: string; endpointUrl: string }>; + ) => Promise<{ + endpointId: number | null; + baseUrl: string; + endpointUrl: string; + }>; }, "resolveStreamingHedgeEndpoint" ); @@ -2604,7 +2770,11 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { return retrySetup.promise; } } - return { endpointId: null, baseUrl: provider.url, endpointUrl: provider.url }; + return { + endpointId: null, + baseUrl: provider.url, + endpointUrl: provider.url, + }; }); const signatureError = new UpstreamProxyError( @@ -2642,7 +2812,11 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { expect(mocks.pickDiscoveryProviders).toHaveBeenCalledTimes(1); expect(doForward).toHaveBeenCalledTimes(2); - retrySetup.resolve({ endpointId: null, baseUrl: sticky.url, endpointUrl: sticky.url }); + retrySetup.resolve({ + endpointId: null, + baseUrl: sticky.url, + endpointUrl: sticky.url, + }); await vi.advanceTimersByTimeAsync(0); const response = await responsePromise; expect(await response.text()).toContain('"normal"'); @@ -2657,8 +2831,16 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { vi.useFakeTimers(); try { const sticky = createProvider({ id: 1, name: "sticky", priority: 1 }); - const normalOne = createProvider({ id: 2, name: "normal-one", priority: 1 }); - const normalTwo = createProvider({ id: 3, name: "normal-two", priority: 1 }); + const normalOne = createProvider({ + id: 2, + name: "normal-one", + priority: 1, + }); + const normalTwo = createProvider({ + id: 3, + name: "normal-two", + priority: 1, + }); const session = createSession(); session.authState = { success: true, @@ -2668,7 +2850,10 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { } as typeof session.authState; session.request.message.messages = [ { role: "user", content: "first" }, - { role: "assistant", content: [{ type: "thinking", thinking: "t", signature: "sig" }] }, + { + role: "assistant", + content: [{ type: "thinking", thinking: "t", signature: "sig" }], + }, ]; setProviderWithSessionRef(session, sticky); session.setSessionBindingSnapshot({ @@ -2703,7 +2888,11 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { resolveStreamingHedgeEndpoint: ( session: ProxySession, provider: Provider - ) => Promise<{ endpointId: number | null; baseUrl: string; endpointUrl: string }>; + ) => Promise<{ + endpointId: number | null; + baseUrl: string; + endpointUrl: string; + }>; }, "resolveStreamingHedgeEndpoint" ).mockImplementation(async (_attemptSession, provider) => { @@ -2711,7 +2900,11 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { retrySetupStarted.resolve(); return retrySetup.promise; } - return { endpointId: null, baseUrl: provider.url, endpointUrl: provider.url }; + return { + endpointId: null, + baseUrl: provider.url, + endpointUrl: provider.url, + }; }); const signatureError = new UpstreamProxyError( @@ -2786,8 +2979,16 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { vi.useFakeTimers(); try { const sticky = createProvider({ id: 1, name: "sticky", priority: 1 }); - const normalOne = createProvider({ id: 2, name: "normal-one", priority: 1 }); - const normalTwo = createProvider({ id: 3, name: "normal-two", priority: 1 }); + const normalOne = createProvider({ + id: 2, + name: "normal-one", + priority: 1, + }); + const normalTwo = createProvider({ + id: 3, + name: "normal-two", + priority: 1, + }); const session = createSession(); session.authState = { success: true, @@ -3246,8 +3447,16 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { vi.useFakeTimers(); try { const sticky = createProvider({ id: 1, name: "sticky", priority: 1 }); - const roundOne = createProvider({ id: 2, name: "round-one", priority: 1 }); - const roundTwo = createProvider({ id: 3, name: "round-two", priority: 1 }); + const roundOne = createProvider({ + id: 2, + name: "round-one", + priority: 1, + }); + const roundTwo = createProvider({ + id: 3, + name: "round-two", + priority: 1, + }); const session = createSession(); session.authState = { success: true, @@ -3396,8 +3605,16 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { let endpointResolver: ReturnType | null = null; try { const sticky = createProvider({ id: 1, name: "sticky", priority: 1 }); - const setupFailure = createProvider({ id: 2, name: "setup-failure", priority: 1 }); - const replacement = createProvider({ id: 3, name: "replacement", priority: 1 }); + const setupFailure = createProvider({ + id: 2, + name: "setup-failure", + priority: 1, + }); + const replacement = createProvider({ + id: 3, + name: "replacement", + priority: 1, + }); const session = createSession(); session.authState = { success: true, @@ -3434,7 +3651,11 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { resolveStreamingHedgeEndpoint: ( session: ProxySession, provider: Provider - ) => Promise<{ endpointId: number | null; baseUrl: string; endpointUrl: string }>; + ) => Promise<{ + endpointId: number | null; + baseUrl: string; + endpointUrl: string; + }>; }, "resolveStreamingHedgeEndpoint" ); @@ -3487,9 +3708,17 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { }); test("Discovery keeps refilling the current round when an error replacement fails setup", async () => { - const initialFailure = createProvider({ id: 1, name: "initial-failure", priority: 1 }); + const initialFailure = createProvider({ + id: 1, + name: "initial-failure", + priority: 1, + }); const pending = createProvider({ id: 2, name: "pending", priority: 1 }); - const setupFailure = createProvider({ id: 3, name: "setup-failure", priority: 1 }); + const setupFailure = createProvider({ + id: 3, + name: "setup-failure", + priority: 1, + }); const healthy = createProvider({ id: 4, name: "healthy", priority: 1 }); const session = createSession(); session.authState = { @@ -3518,7 +3747,11 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { resolveStreamingHedgeEndpoint: ( session: ProxySession, provider: Provider - ) => Promise<{ endpointId: number | null; baseUrl: string; endpointUrl: string }>; + ) => Promise<{ + endpointId: number | null; + baseUrl: string; + endpointUrl: string; + }>; }, "resolveStreamingHedgeEndpoint" ); @@ -3621,7 +3854,10 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { ); expect(doForward).toHaveBeenCalledTimes(2); expect(mocks.clearVersionedSessionProvider).toHaveBeenCalledWith( - expect.objectContaining({ providerId: sticky.id, generation: "g-sticky-failure" }), + expect.objectContaining({ + providerId: sticky.id, + generation: "g-sticky-failure", + }), sticky.id, 0 ); @@ -3899,7 +4135,11 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { priority: 1, limitConcurrentSessions: 1, }); - const winner = createProvider({ id: 3, name: "next-round-winner", priority: 1 }); + const winner = createProvider({ + id: 3, + name: "next-round-winner", + priority: 1, + }); const session = createSession(); session.authState = { success: true, @@ -3924,13 +4164,21 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { resolveStreamingHedgeEndpoint: ( session: ProxySession, provider: Provider - ) => Promise<{ endpointId: number | null; baseUrl: string; endpointUrl: string }>; + ) => Promise<{ + endpointId: number | null; + baseUrl: string; + endpointUrl: string; + }>; }, "resolveStreamingHedgeEndpoint" ); endpointResolver.mockImplementation(async (_attemptSession, provider) => { if (provider.id === setup.id) return new Promise(() => {}); - return { endpointId: null, baseUrl: provider.url, endpointUrl: provider.url }; + return { + endpointId: null, + baseUrl: provider.url, + endpointUrl: provider.url, + }; }); let fallbackController: ReadableStreamDefaultController | null = null; @@ -3990,9 +4238,21 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { let endpointResolver: ReturnType | null = null; try { const fallback = createProvider({ id: 1, name: "fallback", priority: 1 }); - const setup = createProvider({ id: 2, name: "cancelled-setup", priority: 1 }); - const stalled = createProvider({ id: 3, name: "stalled-next-wave", priority: 1 }); - const winner = createProvider({ id: 4, name: "error-refill-winner", priority: 1 }); + const setup = createProvider({ + id: 2, + name: "cancelled-setup", + priority: 1, + }); + const stalled = createProvider({ + id: 3, + name: "stalled-next-wave", + priority: 1, + }); + const winner = createProvider({ + id: 4, + name: "error-refill-winner", + priority: 1, + }); const session = createSession(); session.authState = { success: true, @@ -4025,13 +4285,21 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { resolveStreamingHedgeEndpoint: ( session: ProxySession, provider: Provider - ) => Promise<{ endpointId: number | null; baseUrl: string; endpointUrl: string }>; + ) => Promise<{ + endpointId: number | null; + baseUrl: string; + endpointUrl: string; + }>; }, "resolveStreamingHedgeEndpoint" ); endpointResolver.mockImplementation(async (_attemptSession, provider) => { if (provider.id === setup.id) return new Promise(() => {}); - return { endpointId: null, baseUrl: provider.url, endpointUrl: provider.url }; + return { + endpointId: null, + baseUrl: provider.url, + endpointUrl: provider.url, + }; }); const fallbackFailure = Promise.withResolvers(); @@ -4074,7 +4342,11 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { try { const initial = createProvider({ id: 1, name: "initial", priority: 1 }); const peer = createProvider({ id: 2, name: "peer", priority: 1 }); - const nextRound = createProvider({ id: 3, name: "next-round", priority: 1 }); + const nextRound = createProvider({ + id: 3, + name: "next-round", + priority: 1, + }); const stale = createProvider({ id: 4, name: "stale", priority: 1 }); const session = createSession(); session.authState = { @@ -4142,12 +4414,135 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { } }); + test("a round-cancelled attempt cannot win when its upstream ignores abort and resolves late", async () => { + vi.useFakeTimers(); + try { + const fallback = createProvider({ id: 1, name: "fallback", priority: 1 }); + const cancelled = createProvider({ + id: 2, + name: "cancelled", + priority: 10, + }); + const winner = createProvider({ + id: 3, + name: "next-round-winner", + priority: 1, + }); + const session = createSession(); + session.authState = { + success: true, + user: null, + key: { id: 42 }, + apiKey: null, + } as typeof session.authState; + session.setProvider(fallback); + mocks.getCachedSystemSettings.mockResolvedValue({ + discoveryEnabled: true, + discoveryConcurrency: 2, + maxDiscoveryRounds: 2, + discoverySlaMs: 10, + stickySlaMs: 10, + racingTotalTimeoutMs: 100, + stickyTimeoutCooldownMs: 300_000, + }); + mocks.pickDiscoveryProviders + .mockResolvedValueOnce([cancelled]) + .mockResolvedValueOnce([winner]); + + const cancelledResponse = Promise.withResolvers(); + const winnerResponse = Promise.withResolvers(); + const cancelledReader = vi.fn(); + const cancelledAgentRelease = vi.fn(); + let cancelledSignal: AbortSignal | undefined; + const doForward = vi.spyOn( + ProxyForwarder as unknown as { + doForward: (...args: unknown[]) => Promise; + }, + "doForward" + ); + doForward.mockImplementation(async (attemptSession, ...args) => { + const runtime = attemptSession as ProxySession & AttemptRuntime; + const providerId = runtime.provider!.id; + if (providerId === cancelled.id) { + cancelledSignal = args.at(-1) as AbortSignal; + runtime.releaseAgent = cancelledAgentRelease; + return cancelledResponse.promise; + } + if (providerId === winner.id) return winnerResponse.promise; + return new Response(new ReadableStream(), { + headers: { "content-type": "text/event-stream" }, + }); + }); + + let responseSettled = false; + const responsePromise = ProxyForwarder.send(session).then((response) => { + responseSettled = true; + return response; + }); + await vi.advanceTimersByTimeAsync(10); + await vi.advanceTimersByTimeAsync(0); + + expect(cancelledSignal?.aborted).toBe(true); + expect(doForward).toHaveBeenCalledTimes(3); + + cancelledResponse.resolve( + new Response( + new ReadableStream({ + start(controller) { + controller.enqueue( + new TextEncoder().encode( + 'data: {"type":"content_block_delta","delta":{"text":"too-late"}}\n\n' + ) + ); + }, + cancel: cancelledReader, + }), + { headers: { "content-type": "text/event-stream" } } + ) + ); + await vi.advanceTimersByTimeAsync(0); + + expect(responseSettled).toBe(false); + expect(session.provider?.id).not.toBe(cancelled.id); + expect(cancelledReader).toHaveBeenCalledOnce(); + expect(cancelledAgentRelease).toHaveBeenCalledOnce(); + expect( + mocks.releaseProviderSession.mock.calls.filter( + ([providerId]) => providerId === cancelled.id + ) + ).toHaveLength(1); + + winnerResponse.resolve( + new Response('data: {"type":"content_block_delta","delta":{"text":"winner"}}\n\n', { + headers: { "content-type": "text/event-stream" }, + }) + ); + await vi.advanceTimersByTimeAsync(0); + + const response = await responsePromise; + expect(await response.text()).toContain('"winner"'); + expect(session.provider?.id).toBe(winner.id); + expect(cancelledReader).toHaveBeenCalledOnce(); + expect(cancelledAgentRelease).toHaveBeenCalledOnce(); + } finally { + vi.useRealTimers(); + } + }); + test("a fallback failure refills the reserved round without waiting for its stalled selector", async () => { vi.useFakeTimers(); try { const fallback = createProvider({ id: 1, name: "fallback", priority: 1 }); - const firstRoundLoser = createProvider({ id: 2, name: "first-round-loser", priority: 1 }); - const nextRoundWinner = createProvider({ id: 3, name: "next-round-winner", priority: 1 }); + const firstRoundLoser = createProvider({ + id: 2, + name: "first-round-loser", + priority: 1, + }); + const nextRoundWinner = createProvider({ + id: 3, + name: "next-round-winner", + priority: 1, + }); const staleReservedCandidate = createProvider({ id: 4, name: "stale-reserved-candidate", @@ -4248,7 +4643,11 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { resolveStreamingHedgeEndpoint: ( session: ProxySession, provider: Provider - ) => Promise<{ endpointId: number | null; baseUrl: string; endpointUrl: string }>; + ) => Promise<{ + endpointId: number | null; + baseUrl: string; + endpointUrl: string; + }>; }, "resolveStreamingHedgeEndpoint" ); @@ -4283,8 +4682,16 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { }); test("Discovery transfers the Provider session ref when a rectifier retries the same Provider", async () => { - const initial = createProvider({ id: 1, name: "initial", limitConcurrentSessions: 1 }); - const alternative = createProvider({ id: 2, name: "alternative", limitConcurrentSessions: 1 }); + const initial = createProvider({ + id: 1, + name: "initial", + limitConcurrentSessions: 1, + }); + const alternative = createProvider({ + id: 2, + name: "alternative", + limitConcurrentSessions: 1, + }); const session = createSession(); session.authState = { success: true, @@ -4355,7 +4762,11 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { }); test("Discovery keeps a healthy peer when rectifier retry setup fails", async () => { - const initial = createProvider({ id: 1, name: "initial", limitConcurrentSessions: 1 }); + const initial = createProvider({ + id: 1, + name: "initial", + limitConcurrentSessions: 1, + }); const alternative = createProvider({ id: 2, name: "alternative" }); const session = createSession(); session.authState = { @@ -4388,7 +4799,11 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { resolveStreamingHedgeEndpoint: ( session: ProxySession, provider: Provider - ) => Promise<{ endpointId: number | null; baseUrl: string; endpointUrl: string }>; + ) => Promise<{ + endpointId: number | null; + baseUrl: string; + endpointUrl: string; + }>; }, "resolveStreamingHedgeEndpoint" ); @@ -4448,8 +4863,16 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { limitConcurrentSessions: 1, }); const fallback = createProvider({ id: 2, name: "fallback", priority: 1 }); - const nextRound = createProvider({ id: 3, name: "next-round", priority: 1 }); - const unexpected = createProvider({ id: 4, name: "unexpected", priority: 1 }); + const nextRound = createProvider({ + id: 3, + name: "next-round", + priority: 1, + }); + const unexpected = createProvider({ + id: 4, + name: "unexpected", + priority: 1, + }); const session = createSession(); session.authState = { success: true, @@ -4486,7 +4909,11 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { resolveStreamingHedgeEndpoint: ( session: ProxySession, provider: Provider - ) => Promise<{ endpointId: number | null; baseUrl: string; endpointUrl: string }>; + ) => Promise<{ + endpointId: number | null; + baseUrl: string; + endpointUrl: string; + }>; }, "resolveStreamingHedgeEndpoint" ); @@ -4580,7 +5007,11 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { limitConcurrentSessions: 1, }); const fallback = createProvider({ id: 2, name: "fallback", priority: 1 }); - const unexpected = createProvider({ id: 3, name: "unexpected", priority: 1 }); + const unexpected = createProvider({ + id: 3, + name: "unexpected", + priority: 1, + }); const session = createSession(); session.authState = { success: true, @@ -4616,7 +5047,11 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { resolveStreamingHedgeEndpoint: ( session: ProxySession, provider: Provider - ) => Promise<{ endpointId: number | null; baseUrl: string; endpointUrl: string }>; + ) => Promise<{ + endpointId: number | null; + baseUrl: string; + endpointUrl: string; + }>; }, "resolveStreamingHedgeEndpoint" ); @@ -4688,7 +5123,7 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { expect(peekDeferredStreamingFinalization(session)).toEqual( expect.objectContaining({ bindingIntent: "none", - requiresCompletionMarker: true, + requiresCompletionMarkerForBinding: false, }) ); expect(await response.text()).toContain('"fallback"'); @@ -4701,7 +5136,11 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { test("the Discovery deadline releases a stalled rectifier retry reservation exactly once", async () => { vi.useFakeTimers(); try { - const provider = createProvider({ id: 1, name: "retrying", limitConcurrentSessions: 1 }); + const provider = createProvider({ + id: 1, + name: "retrying", + limitConcurrentSessions: 1, + }); const session = createSession(); session.authState = { success: true, @@ -4735,7 +5174,11 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { resolveStreamingHedgeEndpoint: ( session: ProxySession, provider: Provider - ) => Promise<{ endpointId: number | null; baseUrl: string; endpointUrl: string }>; + ) => Promise<{ + endpointId: number | null; + baseUrl: string; + endpointUrl: string; + }>; }, "resolveStreamingHedgeEndpoint" ); @@ -4745,7 +5188,11 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { retrySetupStarted.resolve(); return retrySetup.promise; } - return { endpointId: null, baseUrl: provider.url, endpointUrl: provider.url }; + return { + endpointId: null, + baseUrl: provider.url, + endpointUrl: provider.url, + }; }); const signatureError = new UpstreamProxyError( @@ -4773,7 +5220,11 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { expect(mocks.releaseProviderSession).toHaveBeenCalledTimes(1); expect(session.hasProviderSessionRef(provider.id)).toBe(false); - retrySetup.resolve({ endpointId: null, baseUrl: provider.url, endpointUrl: provider.url }); + retrySetup.resolve({ + endpointId: null, + baseUrl: provider.url, + endpointUrl: provider.url, + }); await vi.advanceTimersByTimeAsync(0); expect(doForward).toHaveBeenCalledTimes(1); expect(mocks.releaseProviderSession).toHaveBeenCalledTimes(1); @@ -4786,7 +5237,11 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { vi.useFakeTimers(); try { const clientAbort = new AbortController(); - const provider = createProvider({ id: 1, name: "retrying", limitConcurrentSessions: 1 }); + const provider = createProvider({ + id: 1, + name: "retrying", + limitConcurrentSessions: 1, + }); const session = createSession(clientAbort.signal); session.authState = { success: true, @@ -4820,7 +5275,11 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { resolveStreamingHedgeEndpoint: ( session: ProxySession, provider: Provider - ) => Promise<{ endpointId: number | null; baseUrl: string; endpointUrl: string }>; + ) => Promise<{ + endpointId: number | null; + baseUrl: string; + endpointUrl: string; + }>; }, "resolveStreamingHedgeEndpoint" ); @@ -4830,7 +5289,11 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { retrySetupStarted.resolve(); return retrySetup.promise; } - return { endpointId: null, baseUrl: provider.url, endpointUrl: provider.url }; + return { + endpointId: null, + baseUrl: provider.url, + endpointUrl: provider.url, + }; }); const signatureError = new UpstreamProxyError( @@ -4859,7 +5322,11 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { expect(mocks.releaseProviderSession).toHaveBeenCalledTimes(1); expect(session.hasProviderSessionRef(provider.id)).toBe(false); - retrySetup.resolve({ endpointId: null, baseUrl: provider.url, endpointUrl: provider.url }); + retrySetup.resolve({ + endpointId: null, + baseUrl: provider.url, + endpointUrl: provider.url, + }); await vi.advanceTimersByTimeAsync(0); expect(doForward).toHaveBeenCalledTimes(1); expect(mocks.releaseProviderSession).toHaveBeenCalledTimes(1); @@ -4882,7 +5349,11 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { expectedStatus: null, }, ])("rectifier retry setup preserves $label fail-fast semantics", async (scenario) => { - const retrying = createProvider({ id: 1, name: "retrying", limitConcurrentSessions: 1 }); + const retrying = createProvider({ + id: 1, + name: "retrying", + limitConcurrentSessions: 1, + }); const peer = createProvider({ id: 2, name: "peer" }); const unexpected = createProvider({ id: 3, name: "unexpected" }); const session = createSession(); @@ -4915,7 +5386,11 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { resolveStreamingHedgeEndpoint: ( session: ProxySession, provider: Provider - ) => Promise<{ endpointId: number | null; baseUrl: string; endpointUrl: string }>; + ) => Promise<{ + endpointId: number | null; + baseUrl: string; + endpointUrl: string; + }>; }, "resolveStreamingHedgeEndpoint" ); @@ -4924,7 +5399,11 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { retryingEndpointCalls += 1; if (retryingEndpointCalls > 1) throw scenario.setupError; } - return { endpointId: null, baseUrl: provider.url, endpointUrl: provider.url }; + return { + endpointId: null, + baseUrl: provider.url, + endpointUrl: provider.url, + }; }); const signatureError = new UpstreamProxyError("Invalid `signature` in `thinking` block", 400, { @@ -4961,7 +5440,11 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { }); test("Discovery releases a transferred Provider session ref exactly once when rectifier retry setup fails", async () => { - const provider = createProvider({ id: 1, name: "initial", limitConcurrentSessions: 1 }); + const provider = createProvider({ + id: 1, + name: "initial", + limitConcurrentSessions: 1, + }); const session = createSession(); session.authState = { success: true, @@ -4988,7 +5471,11 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { resolveStreamingHedgeEndpoint: ( session: ProxySession, provider: Provider - ) => Promise<{ endpointId: number | null; baseUrl: string; endpointUrl: string }>; + ) => Promise<{ + endpointId: number | null; + baseUrl: string; + endpointUrl: string; + }>; }, "resolveStreamingHedgeEndpoint" ); @@ -5032,7 +5519,11 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { vi.useFakeTimers(); try { const initial = createProvider({ id: 1, name: "initial" }); - const delayed = createProvider({ id: 2, name: "delayed", limitConcurrentSessions: 1 }); + const delayed = createProvider({ + id: 2, + name: "delayed", + limitConcurrentSessions: 1, + }); const session = createSession(); session.authState = { success: true, @@ -5084,7 +5575,12 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { const responsePromise = ProxyForwarder.send(session); await vi.advanceTimersByTimeAsync(10); - resolveAdmission({ allowed: true, count: 1, tracked: true, referenced: true }); + resolveAdmission({ + allowed: true, + count: 1, + tracked: true, + referenced: true, + }); await vi.advanceTimersByTimeAsync(1); const response = await responsePromise; expect(await response.text()).toContain('"winner"'); @@ -5125,7 +5621,9 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { doForward.mockImplementationOnce( async (_attemptSession, _provider, _baseUrl, _audit, _count, _stream, signal) => await new Promise((_resolve, reject) => { - signal?.addEventListener("abort", () => reject(signal.reason), { once: true }); + signal?.addEventListener("abort", () => reject(signal.reason), { + once: true, + }); }) ); @@ -5179,7 +5677,11 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { const clientAbortController = new AbortController(); const addSpy = vi.spyOn(clientAbortController.signal, "addEventListener"); const removeSpy = vi.spyOn(clientAbortController.signal, "removeEventListener"); - const provider = createProvider({ id: 1, name: "p1", firstByteTimeoutStreamingMs: 100 }); + const provider = createProvider({ + id: 1, + name: "p1", + firstByteTimeoutStreamingMs: 100, + }); const session = createSession(clientAbortController.signal); setProviderWithSessionRef(session, provider); session.forwardedRequestBody = "x".repeat(512 * 1024); @@ -5214,7 +5716,11 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { const clientAbortController = new AbortController(); clientAbortController.abort(new Error("client_cancelled")); const addSpy = vi.spyOn(clientAbortController.signal, "addEventListener"); - const provider = createProvider({ id: 1, name: "p1", firstByteTimeoutStreamingMs: 100 }); + const provider = createProvider({ + id: 1, + name: "p1", + firstByteTimeoutStreamingMs: 100, + }); const session = createSession(clientAbortController.signal); setProviderWithSessionRef(session, provider); @@ -5225,7 +5731,9 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { "doForward" ); - await expect(ProxyForwarder.send(session)).rejects.toMatchObject({ statusCode: 499 }); + await expect(ProxyForwarder.send(session)).rejects.toMatchObject({ + statusCode: 499, + }); expect(doForward).not.toHaveBeenCalled(); expect(addSpy.mock.calls.filter(([type]) => type === "abort")).toHaveLength(0); }); diff --git a/tests/unit/proxy/response-handler-client-abort-drain.test.ts b/tests/unit/proxy/response-handler-client-abort-drain.test.ts index a2b0096bc..9d0b093ce 100644 --- a/tests/unit/proxy/response-handler-client-abort-drain.test.ts +++ b/tests/unit/proxy/response-handler-client-abort-drain.test.ts @@ -73,7 +73,9 @@ vi.mock("@/lib/async-task-manager", () => ({ })); vi.mock("@/lib/config/system-settings-cache", () => ({ - getCachedSystemSettings: vi.fn(async () => ({ billNonSuccessfulRequests: false })), + getCachedSystemSettings: vi.fn(async () => ({ + billNonSuccessfulRequests: false, + })), })); vi.mock("@/lib/langfuse/emit-proxy-trace", () => ({ @@ -149,7 +151,10 @@ vi.mock("@/lib/session-manager", () => ({ storeSessionUpstreamResponseMeta: vi.fn(), updateSessionProvider: vi.fn(), updateSessionUsage: vi.fn(), - updateSessionBindingSmart: vi.fn(async () => ({ updated: false, reason: "test" })), + updateSessionBindingSmart: vi.fn(async () => ({ + updated: false, + reason: "test", + })), updateSessionWithCodexCacheKey: vi.fn(), }, })); @@ -295,7 +300,11 @@ function createSession( userAgent: "Go-http-client/1.1", userName: "admin", addProviderToChain(this: ProxySession & { providerChain: unknown[] }, prov: Provider, meta) { - this.providerChain.push({ id: prov.id, name: prov.name, ...(meta ?? {}) }); + this.providerChain.push({ + id: prov.id, + name: prov.name, + ...(meta ?? {}), + }); }, clearResponseTimeout: vi.fn(), getContext1mApplied: () => false, @@ -1642,7 +1651,10 @@ describe("ProxyResponseHandler stream client abort finalization", () => { await expectAllFulfilled(tasks); expect(updateMessageRequestDetailsDurably).toHaveBeenCalledWith( 123, - expect.objectContaining({ durationMs: expect.any(Number), statusCode: 200 }), + expect.objectContaining({ + durationMs: expect.any(Number), + statusCode: 200, + }), expect.objectContaining({ onCommitted: expect.any(Function) }) ); } finally { @@ -1788,7 +1800,10 @@ describe("ProxyResponseHandler stream client abort finalization", () => { expect(updateMessageRequestDuration).not.toHaveBeenCalled(); expect(updateMessageRequestDetailsDurably).toHaveBeenCalledWith( 123, - expect.objectContaining({ durationMs: expect.any(Number), statusCode: 200 }), + expect.objectContaining({ + durationMs: expect.any(Number), + statusCode: 200, + }), expect.objectContaining({ onCommitted: expect.any(Function) }) ); }); @@ -2107,7 +2122,9 @@ describe("ProxyResponseHandler stream client abort finalization", () => { const controller = new AbortController(); controller.abort(); const session = createSession(controller.signal); - Object.assign(session, { sessionId: `session-client-abort-${bindingIntent}` }); + Object.assign(session, { + sessionId: `session-client-abort-${bindingIntent}`, + }); session.recordProviderSessionRef(1); vi.mocked(SessionManager.extractCodexPromptCacheKey).mockReturnValue( "client-abort-cache-key" @@ -2130,7 +2147,7 @@ describe("ProxyResponseHandler stream client abort finalization", () => { providerId, generation: `${bindingIntent}-generation`, }, - requiresCompletionMarker: true, + requiresCompletionMarkerForBinding: true, discoveryLease: { sessionId: `session-client-abort-${bindingIntent}`, keyId: 2, @@ -2227,7 +2244,9 @@ describe("ProxyResponseHandler stream client abort finalization", () => { ); // Must NOT have been recorded as a billed 200 success. const calls = ( - updateMessageRequestDetailsDurably as unknown as { mock: { calls: unknown[][] } } + updateMessageRequestDetailsDurably as unknown as { + mock: { calls: unknown[][] }; + } ).mock.calls; const recorded = calls.find((c) => (c[0] as number) === 123)?.[1] as | { statusCode?: number } @@ -2965,7 +2984,9 @@ describe("ProxyResponseHandler stream client abort finalization", () => { expect(recordFailure).toHaveBeenCalledTimes(1); expect(recordFailure).toHaveBeenCalledWith( 1, - expect.objectContaining({ message: "FAKE_200_JSON_ERROR_MESSAGE_NON_EMPTY" }) + expect.objectContaining({ + message: "FAKE_200_JSON_ERROR_MESSAGE_NON_EMPTY", + }) ); }); @@ -3130,7 +3151,9 @@ describe("ProxyResponseHandler stream client abort finalization", () => { expect(recordFailure).toHaveBeenCalledTimes(1); expect(recordFailure).toHaveBeenCalledWith( 1, - expect.objectContaining({ message: "FAKE_200_JSON_ERROR_MESSAGE_NON_EMPTY" }) + expect.objectContaining({ + message: "FAKE_200_JSON_ERROR_MESSAGE_NON_EMPTY", + }) ); }); @@ -3186,7 +3209,10 @@ describe("ProxyResponseHandler stream client abort finalization", () => { expect(updateMessageRequestDuration).not.toHaveBeenCalled(); expect(updateMessageRequestDetailsDurably).toHaveBeenCalledWith( 123, - expect.objectContaining({ durationMs: expect.any(Number), statusCode: 200 }), + expect.objectContaining({ + durationMs: expect.any(Number), + statusCode: 200, + }), expect.objectContaining({ onCommitted: expect.any(Function) }) ); }); @@ -3218,7 +3244,7 @@ describe("ProxyResponseHandler stream client abort finalization", () => { providerId: null, generation: "non-sse-generation", }, - requiresCompletionMarker: true, + requiresCompletionMarkerForBinding: true, discoveryLease: { sessionId: "non-sse-gemini-discovery", keyId: 2, @@ -3473,7 +3499,7 @@ describe("ProxyResponseHandler stream client abort finalization", () => { providerId: null, generation: "discovery-create-generation", }, - requiresCompletionMarker: true, + requiresCompletionMarkerForBinding: true, discoveryLease: { sessionId: "stream-discovery-cache-binding", keyId: 2, @@ -3523,7 +3549,7 @@ describe("ProxyResponseHandler stream client abort finalization", () => { providerId: null, generation: "stale-discovery-generation", }, - requiresCompletionMarker: true, + requiresCompletionMarkerForBinding: true, discoveryLease: { sessionId: "stream-discovery-cache-conflict", keyId: 2, @@ -3571,7 +3597,7 @@ describe("ProxyResponseHandler stream client abort finalization", () => { providerId: null, generation: "incomplete-discovery-generation", }, - requiresCompletionMarker: true, + requiresCompletionMarkerForBinding: true, discoveryLease: { sessionId: "stream-discovery-cache-incomplete", keyId: 2, diff --git a/tests/unit/proxy/response-handler-endpoint-circuit-isolation.test.ts b/tests/unit/proxy/response-handler-endpoint-circuit-isolation.test.ts index 1de1404bb..1a374fbf5 100644 --- a/tests/unit/proxy/response-handler-endpoint-circuit-isolation.test.ts +++ b/tests/unit/proxy/response-handler-endpoint-circuit-isolation.test.ts @@ -88,6 +88,7 @@ vi.mock("@/lib/session-manager", () => ({ updateSessionBindingSmart: vi.fn(), updateSessionProvider: vi.fn(), updateSessionWithCodexCacheKey: vi.fn(), + storeSessionSpecialSettings: vi.fn(), }, })); @@ -178,8 +179,18 @@ function createSession(opts?: { sessionId?: string | null }): ProxySession { dailyResetMode: "fixed", }; - const user = { id: 123, name: "test-user", dailyResetTime: "00:00", dailyResetMode: "fixed" }; - const key = { id: 456, name: "test-key", dailyResetTime: "00:00", dailyResetMode: "fixed" }; + const user = { + id: 123, + name: "test-user", + dailyResetTime: "00:00", + dailyResetMode: "fixed", + }; + const key = { + id: 456, + name: "test-key", + dailyResetTime: "00:00", + dailyResetMode: "fixed", + }; Object.assign(session, { request: { message: {}, log: "(test)", model: "test-model" }, @@ -369,7 +380,9 @@ function createMisleadingCompletionTextResponse(): Response { const sseText = `event: content_block_delta\ndata: ${JSON.stringify({ type: "content_block_delta", - delta: { text: "the words message_stop and response.completed are ordinary content" }, + delta: { + text: "the words message_stop and response.completed are ordinary content", + }, })}\n\n` + `event: message_delta\ndata: ${JSON.stringify({ type: "message_delta", @@ -495,6 +508,7 @@ function setupCommonMocks() { reason: "test", }); vi.mocked(SessionManager.updateSessionProvider).mockResolvedValue(undefined); + vi.mocked(SessionManager.storeSessionSpecialSettings).mockResolvedValue(undefined); vi.mocked(RateLimitService.trackCost).mockResolvedValue(undefined); vi.mocked(RateLimitService.trackUserDailyCost).mockResolvedValue(undefined); vi.mocked(RateLimitService.decrementLeaseBudget).mockResolvedValue({ @@ -781,7 +795,7 @@ describe("Endpoint circuit breaker isolation", () => { expect(RateLimitService.releaseProviderSession).toHaveBeenCalledWith(1, "fake-session"); }); - it("records a Discovery fallback without a completion marker as failed", async () => { + it("keeps a naturally completed Discovery fallback successful without a completion marker", async () => { const session = createSession(); setDeferredStreamingFinalization(session, { providerId: 1, @@ -795,7 +809,7 @@ describe("Endpoint circuit breaker isolation", () => { endpointUrl: "https://api.test.com", upstreamStatusCode: 200, bindingIntent: "none", - requiresCompletionMarker: true, + requiresCompletionMarkerForBinding: false, }); const clientResponse = await ProxyResponseHandler.dispatch( @@ -805,22 +819,29 @@ describe("Endpoint circuit breaker isolation", () => { await clientResponse.text(); await drainAsyncTasks(); - expect(updateMessageRequestDetailsDurably).toHaveBeenCalledWith( - 1, + const details = vi.mocked(updateMessageRequestDetailsDurably).mock.calls.at(-1)?.[1]; + expect(details).toEqual( expect.objectContaining({ - statusCode: 502, - errorMessage: "STREAM_COMPLETION_MARKER_MISSING", - }), - expect.objectContaining({ onCommitted: expect.any(Function) }) + statusCode: 200, + inputTokens: 100, + outputTokens: 50, + }) ); - expect(mockRecordFailure).toHaveBeenCalledOnce(); - expect(mockRecordSuccess).not.toHaveBeenCalled(); + expect(details).not.toHaveProperty("errorMessage"); + expect(mockRecordFailure).not.toHaveBeenCalled(); + expect(mockRecordSuccess).toHaveBeenCalledWith(1); expect(SessionManager.compareAndSetSessionProvider).not.toHaveBeenCalled(); expect(SessionManager.clearVersionedSessionProvider).not.toHaveBeenCalled(); }); - it("does not clear a create tombstone when the completion marker is missing", async () => { + it("keeps a create tombstone and skips Sticky when the completion marker is missing", async () => { const session = createSession(); + const appendRoutingTraceEvent = vi.fn(); + Object.assign(session, { + appendRoutingTraceEvent, + getRoutingTrace: () => null, + }); + session.recordProviderSessionRef(1); setDeferredStreamingFinalization(session, { providerId: 1, providerName: "test-provider", @@ -839,7 +860,15 @@ describe("Endpoint circuit breaker isolation", () => { providerId: null, generation: "incomplete-generation", }, - requiresCompletionMarker: true, + requiresCompletionMarkerForBinding: true, + discoveryLease: { + sessionId: "fake-session", + keyId: 456, + ownerToken: "missing-marker-owner", + ttlSeconds: 30, + }, + providerSessionRefOwned: true, + providerSessionRefRetainOnSuccess: true, }); const clientResponse = await ProxyResponseHandler.dispatch( @@ -852,8 +881,139 @@ describe("Endpoint circuit breaker isolation", () => { expect(SessionManager.clearVersionedSessionProvider).not.toHaveBeenCalled(); expect(SessionManager.clearSessionProvider).not.toHaveBeenCalled(); expect(SessionManager.compareAndSetSessionProvider).not.toHaveBeenCalled(); + expect(mockRecordFailure).not.toHaveBeenCalled(); + expect(mockRecordSuccess).toHaveBeenCalledWith(1); + expect(RateLimitService.releaseProviderSession).toHaveBeenCalledWith(1, "fake-session"); + expect(appendRoutingTraceEvent).toHaveBeenCalledWith( + expect.objectContaining({ + type: "binding_finalized", + bindingAction: "create", + outcome: "skipped", + reason: "completion_marker_missing", + }) + ); + const details = vi.mocked(updateMessageRequestDetailsDurably).mock.calls.at(-1)?.[1]; + expect(details).toEqual(expect.objectContaining({ statusCode: 200 })); + expect(details).not.toHaveProperty("errorMessage"); }); + it("clears only the captured renew binding when a successful stream has no completion marker", async () => { + const session = createSession(); + const snapshot = { + sessionId: "fake-session", + keyId: 456, + providerId: 1, + generation: "renew-missing-marker-generation", + } as const; + setDeferredStreamingFinalization(session, { + providerId: 1, + providerName: "test-provider", + providerPriority: 10, + attemptNumber: 1, + totalProvidersAttempted: 1, + isFirstAttempt: true, + isFailoverSuccess: false, + endpointId: 42, + endpointUrl: "https://api.test.com", + upstreamStatusCode: 200, + bindingIntent: "renew", + bindingSnapshot: snapshot, + requiresCompletionMarkerForBinding: true, + }); + + const clientResponse = await ProxyResponseHandler.dispatch( + session, + createMisleadingCompletionTextResponse() + ); + await clientResponse.text(); + await drainAsyncTasks(); + + expect(SessionManager.clearVersionedSessionProvider).toHaveBeenCalledWith(snapshot, 1, 0); + expect(SessionManager.clearSessionProvider).not.toHaveBeenCalled(); + expect(SessionManager.compareAndSetSessionProvider).not.toHaveBeenCalled(); + expect(mockRecordFailure).not.toHaveBeenCalled(); + expect(mockRecordSuccess).toHaveBeenCalledWith(1); + const details = vi.mocked(updateMessageRequestDetailsDurably).mock.calls.at(-1)?.[1]; + expect(details).toEqual(expect.objectContaining({ statusCode: 200 })); + expect(details).not.toHaveProperty("errorMessage"); + }); + + it.each([ + { + label: "Claude", + format: "claude" as const, + body: `data: ${JSON.stringify({ + type: "content_block_delta", + delta: { type: "text_delta", text: "ok" }, + })}\n\n`, + }, + { + label: "OpenAI Chat", + format: "openai" as const, + body: `data: ${JSON.stringify({ choices: [{ delta: { content: "ok" } }] })}\n\n`, + }, + { + label: "OpenAI Responses", + format: "response" as const, + body: `event: response.output_text.delta\ndata: ${JSON.stringify({ + type: "response.output_text.delta", + delta: "ok", + })}\n\n`, + }, + { + label: "Gemini NDJSON", + format: "gemini" as const, + body: `${JSON.stringify({ candidates: [{ content: { parts: [{ text: "ok" }] } }] })}\n`, + }, + ])( + "keeps a naturally completed $label stream successful but unbound without a marker", + async ({ format, body }) => { + const session = createSession(); + session.originalFormat = format; + if (format === "gemini" || format === "gemini-cli") { + session.provider = { ...session.provider!, providerType: format }; + } + const snapshot = { + sessionId: "fake-session", + keyId: 456, + providerId: null, + generation: `${format}-natural-eof-generation`, + } as const; + setDeferredStreamingFinalization(session, { + providerId: 1, + providerName: "test-provider", + providerPriority: 10, + attemptNumber: 1, + totalProvidersAttempted: 2, + isFirstAttempt: false, + isFailoverSuccess: true, + endpointId: 42, + endpointUrl: "https://api.test.com", + upstreamStatusCode: 200, + bindingIntent: "create", + bindingSnapshot: snapshot, + requiresCompletionMarkerForBinding: true, + }); + + const clientResponse = await ProxyResponseHandler.dispatch( + session, + new Response(body, { + status: 200, + headers: { "content-type": "text/event-stream" }, + }) + ); + await expect(clientResponse.text()).resolves.toContain("ok"); + await drainAsyncTasks(); + + expect(SessionManager.compareAndSetSessionProvider).not.toHaveBeenCalled(); + expect(mockRecordFailure).not.toHaveBeenCalled(); + expect(mockRecordSuccess).toHaveBeenCalledWith(1); + const details = vi.mocked(updateMessageRequestDetailsDurably).mock.calls.at(-1)?.[1]; + expect(details).toEqual(expect.objectContaining({ statusCode: 200 })); + expect(details).not.toHaveProperty("errorMessage"); + } + ); + it("does not accept completion marker words embedded in ordinary SSE content", async () => { const session = createSession(); setDeferredStreamingFinalization(session, { @@ -874,7 +1034,7 @@ describe("Endpoint circuit breaker isolation", () => { providerId: null, generation: "misleading-content-generation", }, - requiresCompletionMarker: true, + requiresCompletionMarkerForBinding: true, }); const clientResponse = await ProxyResponseHandler.dispatch( @@ -885,14 +1045,11 @@ describe("Endpoint circuit breaker isolation", () => { await drainAsyncTasks(); expect(SessionManager.compareAndSetSessionProvider).not.toHaveBeenCalled(); - expect(updateMessageRequestDetailsDurably).toHaveBeenCalledWith( - 1, - expect.objectContaining({ - statusCode: 502, - errorMessage: "STREAM_COMPLETION_MARKER_MISSING", - }), - expect.objectContaining({ onCommitted: expect.any(Function) }) - ); + expect(mockRecordFailure).not.toHaveBeenCalled(); + expect(mockRecordSuccess).toHaveBeenCalledWith(1); + const details = vi.mocked(updateMessageRequestDetailsDurably).mock.calls.at(-1)?.[1]; + expect(details).toEqual(expect.objectContaining({ statusCode: 200 })); + expect(details).not.toHaveProperty("errorMessage"); }); it("does not let response.done override an earlier nested Responses failure", async () => { @@ -917,7 +1074,7 @@ describe("Endpoint circuit breaker isolation", () => { upstreamStatusCode: 200, bindingIntent: "create", bindingSnapshot: snapshot, - requiresCompletionMarker: true, + requiresCompletionMarkerForBinding: true, }); const body = `event: response.output_text.delta\ndata: ${JSON.stringify({ @@ -944,7 +1101,7 @@ describe("Endpoint circuit breaker isolation", () => { expect(mockRecordSuccess).not.toHaveBeenCalled(); expect(mockRecordFailure).toHaveBeenCalledWith( 1, - expect.objectContaining({ message: "STREAM_COMPLETION_MARKER_MISSING" }) + expect.objectContaining({ message: "UPSTREAM_PROTOCOL_ERROR" }) ); }); @@ -1008,9 +1165,19 @@ describe("Endpoint circuit breaker isolation", () => { response: { candidates: [{ finishReason: "STOP" }] }, })}\n\n`, }, + { + label: "Gemini NDJSON", + format: "gemini" as const, + body: `${JSON.stringify({ + candidates: [{ content: { parts: [{ text: "ok" }] }, finishReason: "STOP" }], + })}\n`, + }, ])("accepts a structurally valid $label completion marker", async ({ format, body }) => { const session = createSession(); session.originalFormat = format; + if (format === "gemini" || format === "gemini-cli") { + session.provider = { ...session.provider!, providerType: format }; + } const snapshot = { sessionId: "fake-session", keyId: 456, @@ -1030,7 +1197,7 @@ describe("Endpoint circuit breaker isolation", () => { upstreamStatusCode: 200, bindingIntent: "create", bindingSnapshot: snapshot, - requiresCompletionMarker: true, + requiresCompletionMarkerForBinding: true, }); const clientResponse = await ProxyResponseHandler.dispatch( @@ -1069,7 +1236,7 @@ describe("Endpoint circuit breaker isolation", () => { upstreamStatusCode: 200, bindingIntent: "create", bindingSnapshot: snapshot, - requiresCompletionMarker: true, + requiresCompletionMarkerForBinding: true, providerSessionRefOwned: true, providerSessionRefRetainOnSuccess: true, }); @@ -1112,7 +1279,7 @@ describe("Endpoint circuit breaker isolation", () => { upstreamStatusCode: 200, bindingIntent: "renew", bindingSnapshot: snapshot, - requiresCompletionMarker: true, + requiresCompletionMarkerForBinding: true, providerSessionRefOwned: true, providerSessionRefRetainOnSuccess: true, }); @@ -1150,7 +1317,7 @@ describe("Endpoint circuit breaker isolation", () => { upstreamStatusCode: 200, bindingIntent: "renew", bindingSnapshot: snapshot, - requiresCompletionMarker: true, + requiresCompletionMarkerForBinding: true, providerSessionRefOwned: true, providerSessionRefRetainOnSuccess: false, }); @@ -1210,7 +1377,7 @@ describe("Endpoint circuit breaker isolation", () => { providerId: null, generation: "lease-guarded-generation", }, - requiresCompletionMarker: true, + requiresCompletionMarkerForBinding: true, discoveryLease: { sessionId: "fake-session", keyId: 456, @@ -1264,7 +1431,7 @@ describe("Endpoint circuit breaker isolation", () => { endpointUrl: "https://api.test.com", upstreamStatusCode: 200, bindingIntent: "create", - requiresCompletionMarker: true, + requiresCompletionMarkerForBinding: true, discoveryLease: { sessionId: "fake-session", keyId: 456, @@ -1311,7 +1478,7 @@ describe("Endpoint circuit breaker isolation", () => { endpointUrl: "https://api.test.com", upstreamStatusCode: 200, bindingIntent: "none", - requiresCompletionMarker: false, + requiresCompletionMarkerForBinding: false, discoveryLease: { sessionId: "fake-session", keyId: 456, @@ -1383,7 +1550,7 @@ describe("Endpoint circuit breaker isolation", () => { upstreamStatusCode: 200, bindingIntent: "create", bindingSnapshot: snapshot, - requiresCompletionMarker: true, + requiresCompletionMarkerForBinding: true, discoveryLease: { sessionId: "fake-session", keyId: 456, @@ -1438,7 +1605,7 @@ describe("Endpoint circuit breaker isolation", () => { upstreamStatusCode: 200, bindingIntent: "renew", bindingSnapshot: snapshot, - requiresCompletionMarker: true, + requiresCompletionMarkerForBinding: true, discoveryLease: { sessionId: "fake-session", keyId: 456, @@ -1501,7 +1668,7 @@ describe("Endpoint circuit breaker isolation", () => { endpointUrl: "https://api.test.com", upstreamStatusCode: 200, bindingIntent: "none", - requiresCompletionMarker: false, + requiresCompletionMarkerForBinding: false, discoveryLease: { sessionId: "fake-session", keyId: 456, @@ -1538,7 +1705,7 @@ describe("Endpoint circuit breaker isolation", () => { endpointUrl: "https://api.test.com", upstreamStatusCode: 200, bindingIntent: "none", - requiresCompletionMarker: false, + requiresCompletionMarkerForBinding: false, discoveryLease: { sessionId: "fake-session", keyId: 456, @@ -1591,7 +1758,7 @@ describe("Endpoint circuit breaker isolation", () => { upstreamStatusCode: 200, bindingIntent: "create", bindingSnapshot: snapshot, - requiresCompletionMarker: true, + requiresCompletionMarkerForBinding: true, providerSessionRefOwned: true, }); session.recordProviderSessionRef(1); diff --git a/tests/unit/repository/message-terminal-write-apis.test.ts b/tests/unit/repository/message-terminal-write-apis.test.ts index 5dbe731f8..6ded61bb9 100644 --- a/tests/unit/repository/message-terminal-write-apis.test.ts +++ b/tests/unit/repository/message-terminal-write-apis.test.ts @@ -1,5 +1,7 @@ import type { StoredCostBreakdown } from "@/types/cost-breakdown"; import type { CreateMessageRequestData } from "@/types/message"; +import type { SQL } from "drizzle-orm"; +import { PgDialect } from "drizzle-orm/pg-core"; import { afterEach, describe, expect, it, vi } from "vitest"; function installSyncBoundaries(insertedRows: readonly Record[] = []) { @@ -25,6 +27,59 @@ function installSyncBoundaries(insertedRows: readonly Record[] return { insertValues, update, updateSet, updateWhere }; } +function installAsyncRoutingTraceBoundaries() { + const controlUpdate = vi.fn(); + const writerUpdateWhere = vi.fn(async (_condition: unknown) => []); + const writerUpdateSet = vi.fn((_values: Record) => ({ + where: writerUpdateWhere, + })); + const writerUpdate = vi.fn((_table: unknown) => ({ set: writerUpdateSet })); + const getMessageWriterDb = vi.fn(() => ({ + update: writerUpdate, + execute: vi.fn(), + })); + const enqueueMessageRequestUpdate = vi.fn(); + const loggerWarn = vi.fn(); + + vi.doMock("@/drizzle/db", () => ({ + db: { + insert: vi.fn(), + update: controlUpdate, + select: vi.fn(), + execute: vi.fn(), + }, + getMessageWriterDb, + })); + vi.doMock("@/lib/config/env.schema", () => ({ + getEnvConfig: vi.fn(() => ({ + MESSAGE_REQUEST_WRITE_MODE: "async" as const, + })), + isDevelopment: vi.fn(() => false), + })); + vi.doMock("@/repository/message-write-buffer", () => ({ + enqueueMessageRequestUpdate, + enqueueMessageRequestUpdateDurably: vi.fn(), + })); + vi.doMock("@/lib/logger", () => ({ + logger: { + debug: vi.fn(), + error: vi.fn(), + info: vi.fn(), + warn: loggerWarn, + }, + })); + + return { + controlUpdate, + enqueueMessageRequestUpdate, + getMessageWriterDb, + loggerWarn, + writerUpdate, + writerUpdateSet, + writerUpdateWhere, + }; +} + const BREAKDOWN = { input: "0.01", output: "0.02", @@ -38,8 +93,11 @@ const BREAKDOWN = { describe("message terminal write APIs", () => { afterEach(() => { + vi.useRealTimers(); vi.doUnmock("@/drizzle/db"); vi.doUnmock("@/lib/config/env.schema"); + vi.doUnmock("@/lib/logger"); + vi.doUnmock("@/repository/message-write-buffer"); }); it("creates a request through the repository and returns its public row", async () => { @@ -205,4 +263,105 @@ describe("message terminal write APIs", () => { expect(updateSet.mock.calls[0]?.[0]).not.toHaveProperty("costUsd"); expect(updateWhere).toHaveBeenCalledTimes(1); }); + + it("patches an async finalized routing trace directly through the writer lane", async () => { + vi.resetModules(); + const boundaries = installAsyncRoutingTraceBoundaries(); + const { updateMessageRequestRoutingTrace } = await import("@/repository/message"); + const routingTrace = { + version: 1 as const, + mode: "discovery" as const, + startedAt: 1_000, + updatedAt: 1_100, + discoveryEnabled: true, + eligible: true, + events: [ + { + type: "binding_finalized" as const, + at: 1_100, + elapsedMs: 100, + bindingAction: "create" as const, + outcome: "updated", + }, + ], + }; + + await updateMessageRequestRoutingTrace(707, routingTrace); + + expect(boundaries.enqueueMessageRequestUpdate).not.toHaveBeenCalled(); + expect(boundaries.controlUpdate).not.toHaveBeenCalled(); + expect(boundaries.getMessageWriterDb).toHaveBeenCalledOnce(); + expect(boundaries.writerUpdate).toHaveBeenCalledOnce(); + expect(boundaries.writerUpdateSet).toHaveBeenCalledWith({ + routingTrace, + updatedAt: expect.any(Date), + }); + expect(boundaries.writerUpdateSet.mock.calls[0]?.[0]).not.toHaveProperty("statusCode"); + expect(boundaries.writerUpdateSet.mock.calls[0]?.[0]).not.toHaveProperty("costUsd"); + + const where = boundaries.writerUpdateWhere.mock.calls[0]?.[0] as SQL; + const query = new PgDialect().sqlToQuery(where); + expect(query.sql).toContain('"message_request"."id" = $1'); + expect(query.sql).toContain('"message_request"."deleted_at" is null'); + expect(query.sql).not.toContain("status_code"); + }); + + it("retries a transient async routing trace write before succeeding", async () => { + vi.useFakeTimers(); + vi.resetModules(); + const boundaries = installAsyncRoutingTraceBoundaries(); + boundaries.writerUpdateWhere + .mockRejectedValueOnce(new Error("writer temporarily unavailable")) + .mockRejectedValueOnce(new Error("writer still unavailable")) + .mockResolvedValueOnce([]); + const { updateMessageRequestRoutingTrace } = await import("@/repository/message"); + const routingTrace = { + version: 1 as const, + mode: "legacy_serial" as const, + startedAt: 2_000, + updatedAt: 2_100, + discoveryEnabled: false, + eligible: false, + events: [], + }; + + const persistence = updateMessageRequestRoutingTrace(708, routingTrace); + await vi.runAllTimersAsync(); + await expect(persistence).resolves.toBeUndefined(); + + expect(boundaries.writerUpdateWhere).toHaveBeenCalledTimes(3); + expect(boundaries.loggerWarn).not.toHaveBeenCalled(); + }); + + it("keeps exhausted async routing trace persistence best-effort and logs once", async () => { + vi.useFakeTimers(); + vi.resetModules(); + const boundaries = installAsyncRoutingTraceBoundaries(); + boundaries.writerUpdateWhere.mockRejectedValue(new Error("writer unavailable")); + const { updateMessageRequestRoutingTrace } = await import("@/repository/message"); + const routingTrace = { + version: 1 as const, + mode: "legacy_serial" as const, + startedAt: 3_000, + updatedAt: 3_100, + discoveryEnabled: false, + eligible: false, + events: [], + }; + + const persistence = updateMessageRequestRoutingTrace(709, routingTrace); + await vi.runAllTimersAsync(); + await expect(persistence).resolves.toBeUndefined(); + + expect(boundaries.writerUpdateWhere).toHaveBeenCalledTimes(3); + expect(boundaries.loggerWarn).toHaveBeenCalledOnce(); + expect(boundaries.loggerWarn).toHaveBeenCalledWith( + "[MessageRequest] Failed to patch finalized routing trace", + { + requestId: 709, + attempts: 3, + error: "writer unavailable", + } + ); + }); }); From 0e313971bdaecd5b8f9b6e53b51b141bf6396bd7 Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Tue, 21 Jul 2026 06:21:28 -0400 Subject: [PATCH 04/12] fix(discovery): address follow-up review findings --- src/app/v1/_lib/proxy/discovery-validity.ts | 1 + src/app/v1/_lib/proxy/error-handler.ts | 6 ++- src/lib/redis/live-chain-store.ts | 2 +- src/lib/validation/discovery-settings.ts | 4 +- tests/unit/lib/redis/live-chain-store.test.ts | 45 +++++++++++++++++++ .../lib/session-manager-binding-smart.test.ts | 8 +++- tests/unit/proxy/discovery-validity.test.ts | 9 ++++ .../error-handler-durable-persistence.test.ts | 21 ++++++++- .../system-settings-discovery.test.ts | 8 ++++ 9 files changed, 99 insertions(+), 5 deletions(-) create mode 100644 tests/unit/lib/redis/live-chain-store.test.ts diff --git a/src/app/v1/_lib/proxy/discovery-validity.ts b/src/app/v1/_lib/proxy/discovery-validity.ts index 41481d800..d3da1a605 100644 --- a/src/app/v1/_lib/proxy/discovery-validity.ts +++ b/src/app/v1/_lib/proxy/discovery-validity.ts @@ -137,6 +137,7 @@ function classifyJson(value: unknown, protocol: DiscoveryProtocol): DiscoveryVal ready: (object.type === "response.output_text.delta" && hasContent(object.delta)) || (object.type === "response.function_call_arguments.delta" && hasContent(object.delta)) || + (object.type === "response.reasoning_summary_text.delta" && hasContent(object.delta)) || (object.type === "response.output_item.added" && hasOpenAIResponsesOutputItem(object.item)), terminal: false, error: false, diff --git a/src/app/v1/_lib/proxy/error-handler.ts b/src/app/v1/_lib/proxy/error-handler.ts index 694233021..97d4232b0 100644 --- a/src/app/v1/_lib/proxy/error-handler.ts +++ b/src/app/v1/_lib/proxy/error-handler.ts @@ -674,7 +674,11 @@ export class ProxyErrorHandler { // 记录请求结束 ProxyErrorHandler.endRequestTracking(session); - void session.closeLiveObservability(); + void session.closeLiveObservability().catch((error) => { + logger.warn("ProxyErrorHandler: Failed to close live observability", { + error: error instanceof Error ? error.message : String(error), + }); + }); } private static endRequestTracking(session: ProxySession): void { diff --git a/src/lib/redis/live-chain-store.ts b/src/lib/redis/live-chain-store.ts index 114545632..7429bff9f 100644 --- a/src/lib/redis/live-chain-store.ts +++ b/src/lib/redis/live-chain-store.ts @@ -32,7 +32,7 @@ function buildKey(sessionId: string, requestSequence: number): string { function inferDiscoveryPhase(trace: RoutingTraceV1): string { const terminalEvent = trace.events.findLast((event) => event.type === "request_finished"); if (terminalEvent) { - switch (trace.summary?.outcome ?? terminalEvent.outcome) { + switch (terminalEvent.outcome ?? trace.summary?.outcome) { case "success": return "completed"; case "client_abort": diff --git a/src/lib/validation/discovery-settings.ts b/src/lib/validation/discovery-settings.ts index 319c6f382..f7f1da63d 100644 --- a/src/lib/validation/discovery-settings.ts +++ b/src/lib/validation/discovery-settings.ts @@ -13,7 +13,9 @@ export const DISCOVERY_FIELD_LIMITS = { export type DiscoverySettingField = keyof typeof DISCOVERY_FIELD_LIMITS; export function isDiscoverySettingField(value: unknown): value is DiscoverySettingField { - return typeof value === "string" && value in DISCOVERY_FIELD_LIMITS; + return ( + typeof value === "string" && Object.prototype.hasOwnProperty.call(DISCOVERY_FIELD_LIMITS, value) + ); } export function getDiscoveryValidationErrorCode( diff --git a/tests/unit/lib/redis/live-chain-store.test.ts b/tests/unit/lib/redis/live-chain-store.test.ts new file mode 100644 index 000000000..40aff5afc --- /dev/null +++ b/tests/unit/lib/redis/live-chain-store.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it, vi } from "vitest"; + +vi.mock("server-only", () => ({})); + +import { inferPhase } from "@/lib/redis/live-chain-store"; +import type { RoutingTraceV1 } from "@/types/routing-trace"; + +describe("live Discovery chain phase", () => { + it("uses the request_finished outcome over an earlier summary outcome", () => { + const trace = { + version: 1, + mode: "discovery", + startedAt: 1_000, + updatedAt: 1_200, + discoveryEnabled: true, + eligible: true, + summary: { + outcome: "success", + statusCode: 200, + }, + events: [ + { + type: "winner_committed", + at: 1_050, + elapsedMs: 50, + round: 1, + attemptId: "attempt-1", + attemptKind: "normal", + provider: { id: 1 }, + outcome: "winner", + statusCode: 200, + }, + { + type: "request_finished", + at: 1_200, + elapsedMs: 200, + outcome: "failed", + statusCode: 502, + }, + ], + } as RoutingTraceV1; + + expect(inferPhase([], trace)).toBe("failed"); + }); +}); diff --git a/tests/unit/lib/session-manager-binding-smart.test.ts b/tests/unit/lib/session-manager-binding-smart.test.ts index 62c188797..dbc0855b7 100644 --- a/tests/unit/lib/session-manager-binding-smart.test.ts +++ b/tests/unit/lib/session-manager-binding-smart.test.ts @@ -181,7 +181,13 @@ describe("SessionManager.updateSessionBindingSmart forceUpdate", () => { expect(findProviderById).not.toHaveBeenCalled(); expect(isCircuitOpen).not.toHaveBeenCalled(); // forceUpdate goes straight to the persistence path. - expect(findProviderById).not.toHaveBeenCalled(); + expect(bindingMocks.mutateLegacySessionBindingSafely).toHaveBeenCalledWith( + expect.objectContaining({ + sessionId: SID, + keyId: KEY_ID, + mutation: { type: "set", providerId: 2 }, + }) + ); }); it("forceUpdate=true also persists the keyId binding with TTL", async () => { diff --git a/tests/unit/proxy/discovery-validity.test.ts b/tests/unit/proxy/discovery-validity.test.ts index b553694dc..afef74a1a 100644 --- a/tests/unit/proxy/discovery-validity.test.ts +++ b/tests/unit/proxy/discovery-validity.test.ts @@ -110,6 +110,15 @@ describe("discovery validity", () => { ).toBe(true); }); + it("accepts an OpenAI Responses reasoning summary text delta as deliverable content", () => { + expect( + classifyDiscoveryChunk( + 'data: {"type":"response.reasoning_summary_text.delta","delta":"thinking"}\n\n', + "openai-responses" + ).ready + ).toBe(true); + }); + it("holds Responses output-item metadata until a text delta is deliverable", () => { const parser = new DiscoveryValidityParser("openai-responses"); diff --git a/tests/unit/proxy/error-handler-durable-persistence.test.ts b/tests/unit/proxy/error-handler-durable-persistence.test.ts index 803a294d8..e2e4f068d 100644 --- a/tests/unit/proxy/error-handler-durable-persistence.test.ts +++ b/tests/unit/proxy/error-handler-durable-persistence.test.ts @@ -20,6 +20,7 @@ const mocks = vi.hoisted(() => ({ passThroughUpstreamErrorMessage: false, })), updateMessageRequestDetailsDurably: vi.fn(), + loggerWarn: vi.fn(), })); vi.mock("@/lib/error-rule-detector", () => ({ @@ -51,7 +52,7 @@ vi.mock("@/lib/logger", () => ({ fatal: vi.fn(), info: vi.fn(), trace: vi.fn(), - warn: vi.fn(), + warn: mocks.loggerWarn, }, })); @@ -129,6 +130,24 @@ describe("ProxyErrorHandler.handle durable persistence", () => { mocks.updateMessageRequestDetailsDurably.mockResolvedValue(undefined); }); + test("logs a rejected live observability close without leaking an unhandled rejection", async () => { + const session = await createSession(); + attachMessageContext(session); + vi.spyOn(session, "closeLiveObservability").mockRejectedValueOnce( + new Error("redis unavailable") + ); + + const response = await ProxyErrorHandler.handle(session, new Error("fetch failed")); + + expect(response.status).toBe(500); + await vi.waitFor(() => { + expect(mocks.loggerWarn).toHaveBeenCalledWith( + "ProxyErrorHandler: Failed to close live observability", + { error: "redis unavailable" } + ); + }); + }); + test("emits the trace, awaits persistence, then ends status tracking", async () => { const commit = Promise.withResolvers(); mocks.updateMessageRequestDetailsDurably.mockReturnValueOnce(commit.promise); diff --git a/tests/unit/validation/system-settings-discovery.test.ts b/tests/unit/validation/system-settings-discovery.test.ts index 9df378943..3039b1fea 100644 --- a/tests/unit/validation/system-settings-discovery.test.ts +++ b/tests/unit/validation/system-settings-discovery.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from "vitest"; +import { isDiscoverySettingField } from "@/lib/validation/discovery-settings"; import { UpdateSystemSettingsSchema } from "@/lib/validation/schemas"; describe("UpdateSystemSettingsSchema Discovery settings", () => { @@ -68,4 +69,11 @@ describe("UpdateSystemSettingsSchema Discovery settings", () => { discoveryEnabled: true, }); }); + + it("rejects inherited Object prototype names as Discovery fields", () => { + expect(isDiscoverySettingField("toString")).toBe(false); + expect(isDiscoverySettingField("valueOf")).toBe(false); + expect(isDiscoverySettingField("__proto__")).toBe(false); + expect(isDiscoverySettingField("discoverySlaMs")).toBe(true); + }); }); From 0c6df07a64f5bd7cb5d73656783fc6058d937a40 Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Tue, 21 Jul 2026 07:35:04 -0400 Subject: [PATCH 05/12] fix(observability): durably persist final routing trace --- src/repository/message-write-buffer.ts | 352 +++++++++++++++--- src/repository/message.ts | 46 +-- .../message-terminal-write-apis.test.ts | 98 ++--- .../repository/message-write-buffer.test.ts | 327 ++++++++++++++++ 4 files changed, 682 insertions(+), 141 deletions(-) diff --git a/src/repository/message-write-buffer.ts b/src/repository/message-write-buffer.ts index ce6c3309f..c5199893a 100644 --- a/src/repository/message-write-buffer.ts +++ b/src/repository/message-write-buffer.ts @@ -49,6 +49,12 @@ export type MessageRequestUpdateRecord = { export type DurableMessageRequestUpdateOptions = { timeoutMs?: number; onCommitted?: (patch: Readonly) => void | Promise; + /** + * terminal (default) owns the request outcome and only updates an unfinalized row. + * post-terminal-metadata is acknowledged after commit but may update an already + * finalized row; callers must use it only for idempotent metadata patches. + */ + writeScope?: "terminal" | "post-terminal-metadata"; }; type DurableAcknowledgement = { @@ -60,6 +66,7 @@ type DurableAcknowledgement = { settled: boolean; timeoutId: NodeJS.Timeout | null; commitNotified: boolean; + writeScope: NonNullable; onCommittedCallbacks: Set>; }; @@ -80,6 +87,25 @@ type WriterConfig = { const DEFAULT_DURABLE_ACK_TIMEOUT_MS = 120_000; const OVERFLOW_LOG_AGGREGATION_MS = 1_000; +const SHUTDOWN_POST_TERMINAL_FLUSH_ATTEMPTS = 2; + +function resolveDurableAcknowledgementTimeoutMs( + options: DurableMessageRequestUpdateOptions +): number { + const timeoutMs = options.timeoutMs ?? DEFAULT_DURABLE_ACK_TIMEOUT_MS; + return Number.isFinite(timeoutMs) && timeoutMs > 0 ? timeoutMs : DEFAULT_DURABLE_ACK_TIMEOUT_MS; +} + +function durableAcknowledgementTimeoutError(): Error { + return new Error("durable message_request acknowledgement timed out"); +} + +function isRoutingTraceOnlyPatch(patch: MessageRequestUpdatePatch): boolean { + return ( + patch.routingTrace !== undefined && + Object.entries(patch).every(([key, value]) => value === undefined || key === "routingTrace") + ); +} type EvictablePendingEntry = { id: number; @@ -281,7 +307,7 @@ function takeBatch( export function buildBatchUpdateSql( updates: MessageRequestUpdateRecord[], - options: { returnUpdatedIds?: boolean; durableIds?: readonly number[] } = {} + options: { returnUpdatedIds?: boolean; fencedDurableIds?: readonly number[] } = {} ): SQL | null { if (updates.length === 0) { return null; @@ -345,16 +371,16 @@ export function buildBatchUpdateSql( sql`, ` ); const updateIds = new Set(ids); - const durableIds = Array.from(new Set(options.durableIds ?? [])).filter((id) => + const fencedDurableIds = Array.from(new Set(options.fencedDurableIds ?? [])).filter((id) => updateIds.has(id) ); const durableFence = - durableIds.length === 0 + fencedDurableIds.length === 0 ? sql`` - : durableIds.length === ids.length + : fencedDurableIds.length === ids.length ? sql` AND ${sql.identifier("status_code")} IS NULL` : sql` AND (id NOT IN (${sql.join( - durableIds.map((id) => sql`${id}`), + fencedDurableIds.map((id) => sql`${id}`), sql`, ` )}) OR ${sql.identifier("status_code")} IS NULL)`; @@ -402,6 +428,8 @@ function getPatchRetentionPriority(patch: MessageRequestUpdatePatch): number { class MessageRequestWriteBuffer { private readonly config: WriterConfig; private readonly pending = new Map(); + private readonly deferredOrdinary = new Map(); + private readonly postTerminalMetadataTasks = new Map>(); private readonly evictableIndex = new EvictablePendingIndex(); private readonly durableAcknowledgements = new Map(); private flushTimer: NodeJS.Timeout | null = null; @@ -414,6 +442,7 @@ class MessageRequestWriteBuffer { private flushAgainAfterCurrent = false; private flushInFlight: Promise | null = null; private readonly commitCallbacksInFlight = new Set>(); + private stopDrainAcceptingLateMetadata = false; private stopping = false; constructor(config: WriterConfig) { @@ -422,6 +451,18 @@ class MessageRequestWriteBuffer { enqueue(id: number, patch: MessageRequestUpdatePatch): void { const existing = this.pending.get(id); + const activeAcknowledgement = this.durableAcknowledgements.get(id); + const postTerminalAcknowledgement = + (existing?.durableAcknowledgement ?? activeAcknowledgement)?.writeScope === + "post-terminal-metadata" && + !(existing?.durableAcknowledgement ?? activeAcknowledgement)?.settled; + if (postTerminalAcknowledgement) { + // A late ordinary update must never be merged into a trace-only ACK: that + // would let terminal/billing fields bypass the terminal status fence. + const deferred = this.deferredOrdinary.get(id); + this.deferredOrdinary.set(id, mergePatch(deferred ?? {}, patch)); + return; + } // existing is older, patch is newer -> for replacement fields newer wins. this.setPending(id, mergePatch(existing?.patch ?? {}, patch), existing?.durableAcknowledgement); @@ -434,23 +475,26 @@ class MessageRequestWriteBuffer { patch: MessageRequestUpdatePatch, options: DurableMessageRequestUpdateOptions = {} ): Promise { + if (options.writeScope === "post-terminal-metadata") { + return this.enqueuePostTerminalMetadataDurably(id, patch, options); + } if (this.stopping) { return Promise.reject(new Error("message_request writer is stopping")); } const activeAcknowledgement = this.durableAcknowledgements.get(id); if (activeAcknowledgement && !activeAcknowledgement.settled) { - // The first durable claimant owns both the terminal patch and its commit + // The first terminal claimant owns both the terminal patch and its commit // callback. Later contenders may observe its SQL acknowledgement, but // must not merge a contradictory terminal outcome or publish side effects. return activeAcknowledgement.promise.then(() => false); } - if (this.durableAcknowledgements.size >= this.config.maxPending) { return Promise.reject(new Error("durable message_request queue is full")); } - const acknowledgement = this.createDurableAcknowledgement(id, options); + const deadlineAt = Date.now() + resolveDurableAcknowledgementTimeoutMs(options); + const acknowledgement = this.createDurableAcknowledgement(id, options, deadlineAt); const existing = this.pending.get(id); this.setPending(id, mergePatch(existing?.patch ?? {}, patch), acknowledgement); @@ -467,9 +511,137 @@ class MessageRequestWriteBuffer { return acknowledgement.promise.then(() => true); } - private createDurableAcknowledgement( + private enqueuePostTerminalMetadataDurably( id: number, + patch: MessageRequestUpdatePatch, options: DurableMessageRequestUpdateOptions + ): Promise { + if (!isRoutingTraceOnlyPatch(patch)) { + return Promise.reject( + new Error("post-terminal metadata updates may only contain routingTrace") + ); + } + const existingTask = this.postTerminalMetadataTasks.get(id); + if (existingTask) { + // Binding finalization is once-guarded upstream. Coalesce accidental + // duplicate callers so they cannot allocate unbounded waiters/timers. + return existingTask; + } + if ( + this.stopping && + (!this.stopDrainAcceptingLateMetadata || this.commitCallbacksInFlight.size === 0) + ) { + return Promise.reject(new Error("message_request writer is stopping")); + } + if (this.postTerminalMetadataTasks.size >= this.config.maxPending) { + return Promise.reject(new Error("durable message_request queue is full")); + } + + let task: Promise; + task = this.persistPostTerminalMetadataDurably(id, patch, options).finally(() => { + if (this.postTerminalMetadataTasks.get(id) === task) { + this.postTerminalMetadataTasks.delete(id); + } + }); + this.postTerminalMetadataTasks.set(id, task); + return task; + } + + private async persistPostTerminalMetadataDurably( + id: number, + patch: MessageRequestUpdatePatch, + options: DurableMessageRequestUpdateOptions + ): Promise { + const deadlineAt = Date.now() + resolveDurableAcknowledgementTimeoutMs(options); + while (true) { + if (Date.now() >= deadlineAt) throw durableAcknowledgementTimeoutError(); + const activeAcknowledgement = this.durableAcknowledgements.get(id); + if (activeAcknowledgement && !activeAcknowledgement.settled) { + await this.waitForAcknowledgementSettlement(activeAcknowledgement, deadlineAt); + continue; + } + if (this.pending.has(id)) { + await this.flush(); + if (this.pending.has(id)) await this.waitForRetry(deadlineAt); + continue; + } + break; + } + + if (this.durableAcknowledgements.size >= this.config.maxPending) { + throw new Error("durable message_request queue is full"); + } + const acknowledgement = this.createDurableAcknowledgement(id, options, deadlineAt); + this.setPending(id, patch, acknowledgement); + if (!this.enforcePendingLimit()) { + this.deletePending(id); + this.rejectDurableAcknowledgement( + acknowledgement, + new Error("durable message_request queue is full") + ); + } else { + this.scheduleFlushIfNeeded(); + } + + if (this.stopping) { + for ( + let attempt = 0; + attempt < SHUTDOWN_POST_TERMINAL_FLUSH_ATTEMPTS && !acknowledgement.settled; + attempt++ + ) { + await this.flush(); + } + if (!acknowledgement.settled) { + const pending = this.pending.get(id); + if (pending?.durableAcknowledgement === acknowledgement) { + this.deletePending(id); + } + this.rejectDurableAcknowledgement( + acknowledgement, + new Error("post-terminal metadata did not persist during writer shutdown") + ); + } + } + await acknowledgement.promise; + return true; + } + + private async waitForAcknowledgementSettlement( + acknowledgement: DurableAcknowledgement, + deadlineAt: number + ): Promise { + const remainingMs = deadlineAt - Date.now(); + if (remainingMs <= 0) throw durableAcknowledgementTimeoutError(); + let timeoutId: NodeJS.Timeout | null = null; + try { + await Promise.race([ + acknowledgement.promise.then( + () => undefined, + () => undefined + ), + new Promise((_, reject) => { + timeoutId = setTimeout(() => reject(durableAcknowledgementTimeoutError()), remainingMs); + timeoutId.unref?.(); + }), + ]); + } finally { + if (timeoutId) clearTimeout(timeoutId); + } + } + + private async waitForRetry(deadlineAt: number): Promise { + const remainingMs = deadlineAt - Date.now(); + if (remainingMs <= 0) throw durableAcknowledgementTimeoutError(); + await new Promise((resolve) => { + const timeoutId = setTimeout(resolve, Math.min(50, remainingMs)); + timeoutId.unref?.(); + }); + } + + private createDurableAcknowledgement( + id: number, + options: DurableMessageRequestUpdateOptions, + deadlineAt: number ): DurableAcknowledgement { let resolvePromise!: () => void; let rejectPromise!: (error: Error) => void; @@ -486,22 +658,18 @@ class MessageRequestWriteBuffer { settled: false, timeoutId: null, commitNotified: false, + writeScope: options.writeScope ?? "terminal", onCommittedCallbacks: new Set(options.onCommitted ? [options.onCommitted] : []), }; - const timeoutMs = options.timeoutMs ?? DEFAULT_DURABLE_ACK_TIMEOUT_MS; - const effectiveTimeoutMs = - Number.isFinite(timeoutMs) && timeoutMs > 0 ? timeoutMs : DEFAULT_DURABLE_ACK_TIMEOUT_MS; + const remainingMs = Math.max(1, deadlineAt - Date.now()); acknowledgement.timeoutId = setTimeout(() => { const pending = this.pending.get(id); if (pending?.durableAcknowledgement === acknowledgement) { this.deletePending(id); } - this.rejectDurableAcknowledgement( - acknowledgement, - new Error("durable message_request acknowledgement timed out") - ); - }, effectiveTimeoutMs); + this.rejectDurableAcknowledgement(acknowledgement, durableAcknowledgementTimeoutError()); + }, remainingMs); acknowledgement.timeoutId.unref?.(); this.durableAcknowledgements.set(id, acknowledgement); @@ -516,28 +684,22 @@ class MessageRequestWriteBuffer { acknowledgement.commitNotified = true; for (const callback of acknowledgement.onCommittedCallbacks) { - try { - const result = callback(patch); - if (result && typeof result.then === "function") { - let callbackPromise: Promise; - callbackPromise = Promise.resolve(result) - .catch((error: unknown) => { - logger.error("[MessageRequestWriteBuffer] Durable commit callback failed", { - error: error instanceof Error ? error.message : String(error), - messageRequestId: acknowledgement.id, - }); - }) - .finally(() => { - this.commitCallbacksInFlight.delete(callbackPromise); - }); - this.commitCallbacksInFlight.add(callbackPromise); - } - } catch (error) { - logger.error("[MessageRequestWriteBuffer] Durable commit callback failed", { - error: error instanceof Error ? error.message : String(error), - messageRequestId: acknowledgement.id, + // Register the callback before invoking it. A callback may enqueue the + // post-terminal routing trace while the writer is already stopping; the + // shutdown drain must see that work as in-flight and keep accepting it. + let callbackPromise: Promise; + callbackPromise = Promise.resolve() + .then(() => callback(patch)) + .catch((error: unknown) => { + logger.error("[MessageRequestWriteBuffer] Durable commit callback failed", { + error: error instanceof Error ? error.message : String(error), + messageRequestId: acknowledgement.id, + }); + }) + .finally(() => { + this.commitCallbacksInFlight.delete(callbackPromise); }); - } + this.commitCallbacksInFlight.add(callbackPromise); } acknowledgement.onCommittedCallbacks.clear(); } @@ -552,6 +714,7 @@ class MessageRequestWriteBuffer { if (this.durableAcknowledgements.get(acknowledgement.id) === acknowledgement) { this.durableAcknowledgements.delete(acknowledgement.id); } + this.releaseDeferredOrdinary(acknowledgement); acknowledgement.resolve(); } @@ -568,9 +731,29 @@ class MessageRequestWriteBuffer { if (this.durableAcknowledgements.get(acknowledgement.id) === acknowledgement) { this.durableAcknowledgements.delete(acknowledgement.id); } + this.releaseDeferredOrdinary(acknowledgement); acknowledgement.reject(error); } + private releaseDeferredOrdinary(acknowledgement: DurableAcknowledgement): void { + if (acknowledgement.writeScope !== "post-terminal-metadata") { + return; + } + const patch = this.deferredOrdinary.get(acknowledgement.id); + if (!patch) { + return; + } + this.deferredOrdinary.delete(acknowledgement.id); + const existing = this.pending.get(acknowledgement.id); + this.setPending( + acknowledgement.id, + mergePatch(existing?.patch ?? {}, patch), + existing?.durableAcknowledgement + ); + this.enforcePendingLimit(); + this.scheduleFlushIfNeeded(); + } + private rejectAllDurableAcknowledgements(error: Error): void { for (const acknowledgement of this.durableAcknowledgements.values()) { this.rejectDurableAcknowledgement(acknowledgement, error); @@ -730,12 +913,12 @@ class MessageRequestWriteBuffer { const requiresUpdatedIds = batch.some( (item) => item.durableAcknowledgement && !item.durableAcknowledgement.settled ); - const durableIds = batch.flatMap((item) => - item.durableAcknowledgement ? [item.id] : [] + const fencedDurableIds = batch.flatMap((item) => + item.durableAcknowledgement?.writeScope === "terminal" ? [item.id] : [] ); const query = buildBatchUpdateSql(batch, { returnUpdatedIds: requiresUpdatedIds, - durableIds, + fencedDurableIds, }); if (!query) { for (const item of batch) { @@ -779,6 +962,15 @@ class MessageRequestWriteBuffer { continue; } const existing = this.pending.get(item.id); + if ( + !item.durableAcknowledgement && + existing?.durableAcknowledgement?.writeScope === "post-terminal-metadata" && + !existing.durableAcknowledgement.settled + ) { + const deferred = this.deferredOrdinary.get(item.id); + this.deferredOrdinary.set(item.id, mergePatch(item.patch, deferred ?? {})); + continue; + } const durableAcknowledgement = item.durableAcknowledgement && !item.durableAcknowledgement.settled ? item.durableAcknowledgement @@ -823,33 +1015,62 @@ class MessageRequestWriteBuffer { async stop(): Promise { this.stopping = true; + this.stopDrainAcceptingLateMetadata = true; this.clearFlushTimer(); - await this.flush(); - // stop 期间尽量补刷一次，避免极小概率竞态导致的 tail 更新残留 - if (this.pending.size > 0) { + + const flushForShutdown = async (): Promise => { await this.flush(); + // A failed batch is requeued. Give shutdown one bounded retry, matching + // the previous stop behavior without accepting an unbounded retry loop. + if (this.pending.size > 0) await this.flush(); + return this.pending.size === 0; + }; + + let shutdownError: Error | null = null; + if (!(await flushForShutdown())) { + shutdownError = new Error("message_request writer shutdown persistence failed"); } - if (this.pending.size > 0) { - const error = new Error("message_request writer shutdown persistence failed"); - this.rejectAllDurableAcknowledgements(error); - this.clearOverflowLogTimer(); - this.flushOverflowLog(); - this.pending.clear(); - this.evictableIndex.clear(); - throw error; + + // Terminal onCommitted callbacks may perform one acknowledged routing-trace + // patch. That dedicated path actively flushes while stopping, so join the + // callbacks before closing late-metadata admission or clearing the queue. + while ( + !shutdownError && + (this.commitCallbacksInFlight.size > 0 || this.postTerminalMetadataTasks.size > 0) + ) { + await Promise.allSettled([ + ...this.commitCallbacksInFlight, + ...this.postTerminalMetadataTasks.values(), + ]); + } + + this.stopDrainAcceptingLateMetadata = false; + // A callback can settle immediately after its final acknowledged enqueue. + // Drain that tail before deciding whether shutdown completed durably. + if (!shutdownError && !(await flushForShutdown())) { + shutdownError = new Error("message_request writer shutdown persistence failed"); } - if (this.durableAcknowledgements.size > 0) { + + if (shutdownError) { + this.rejectAllDurableAcknowledgements(shutdownError); + } else if (this.durableAcknowledgements.size > 0) { this.rejectAllDurableAcknowledgements( new Error("message_request writer stopped before durable commit") ); } - while (this.commitCallbacksInFlight.size > 0) { - await Promise.allSettled([...this.commitCallbacksInFlight]); + while (this.commitCallbacksInFlight.size > 0 || this.postTerminalMetadataTasks.size > 0) { + await Promise.allSettled([ + ...this.commitCallbacksInFlight, + ...this.postTerminalMetadataTasks.values(), + ]); } this.clearOverflowLogTimer(); this.flushOverflowLog(); this.pending.clear(); + this.deferredOrdinary.clear(); + this.postTerminalMetadataTasks.clear(); this.evictableIndex.clear(); + if (shutdownError) throw shutdownError; } } @@ -889,13 +1110,32 @@ export function enqueueMessageRequestUpdateDurably( new Error("durable message_request buffer API requires async write mode") ); } - const buffer = getBuffer(); + const buffer = + options?.writeScope === "post-terminal-metadata" && _bufferState === "stopping" + ? _buffer + : getBuffer(); if (!buffer) { return Promise.reject(new Error("message_request writer is not running")); } return buffer.enqueueDurably(id, patch, options); } +export function enqueueMessageRequestPostTerminalRoutingTraceDurably( + id: number, + routingTrace: RoutingTraceV1, + options: Omit = {} +): Promise { + const normalized = normalizeRoutingTrace(routingTrace); + if (!normalized) { + return Promise.reject(new Error("post-terminal routing trace is invalid")); + } + return enqueueMessageRequestUpdateDurably( + id, + { routingTrace: normalized }, + { ...options, writeScope: "post-terminal-metadata" } + ); +} + export async function flushMessageRequestWriteBuffer(): Promise { if (!_buffer) { return; diff --git a/src/repository/message.ts b/src/repository/message.ts index e4c187ef1..1ec782675 100644 --- a/src/repository/message.ts +++ b/src/repository/message.ts @@ -20,6 +20,7 @@ import { EXCLUDE_WARMUP_CONDITION } from "./_shared/message-request-conditions"; import { toMessageRequest } from "./_shared/transformers"; import { type DurableMessageRequestUpdateOptions, + enqueueMessageRequestPostTerminalRoutingTraceDurably, enqueueMessageRequestUpdate, enqueueMessageRequestUpdateDurably, type MessageRequestUpdatePatch, @@ -624,33 +625,36 @@ export async function updateMessageRequestRoutingTrace( routingTrace: RoutingTraceV1 ): Promise { const normalized = normalizeRoutingTrace(routingTrace); - if (!normalized) return; - - const useWriterLane = getEnvConfig().MESSAGE_REQUEST_WRITE_MODE === "async"; - const maxAttempts = 3; - let lastError: unknown; + if (!normalized) { + logger.warn("[MessageRequest] Skipped patching invalid routing trace", { + requestId: id, + }); + return; + } - for (let attempt = 0; attempt < maxAttempts; attempt++) { + if (getEnvConfig().MESSAGE_REQUEST_WRITE_MODE === "async") { try { - const traceDb = useWriterLane ? getMessageWriterDb() : db; - await traceDb - .update(messageRequest) - .set({ routingTrace: normalized, updatedAt: new Date() }) - .where(and(eq(messageRequest.id, id), isNull(messageRequest.deletedAt))); - return; + await enqueueMessageRequestPostTerminalRoutingTraceDurably(id, normalized); } catch (error) { - lastError = error; - if (attempt < maxAttempts - 1) { - await new Promise((resolve) => setTimeout(resolve, 50 * (attempt + 1))); - } + logger.warn("[MessageRequest] Failed to patch finalized routing trace", { + requestId: id, + error: error instanceof Error ? error.message : String(error), + }); } + return; } - logger.warn("[MessageRequest] Failed to patch finalized routing trace", { - requestId: id, - attempts: maxAttempts, - error: lastError instanceof Error ? lastError.message : String(lastError), - }); + try { + await db + .update(messageRequest) + .set({ routingTrace: normalized, updatedAt: new Date() }) + .where(and(eq(messageRequest.id, id), isNull(messageRequest.deletedAt))); + } catch (error) { + logger.warn("[MessageRequest] Failed to patch finalized routing trace", { + requestId: id, + error: error instanceof Error ? error.message : String(error), + }); + } } export async function updateMessageRequestDetailsIfUnfinalized( diff --git a/tests/unit/repository/message-terminal-write-apis.test.ts b/tests/unit/repository/message-terminal-write-apis.test.ts index 6ded61bb9..6c14bb8d4 100644 --- a/tests/unit/repository/message-terminal-write-apis.test.ts +++ b/tests/unit/repository/message-terminal-write-apis.test.ts @@ -1,7 +1,5 @@ import type { StoredCostBreakdown } from "@/types/cost-breakdown"; import type { CreateMessageRequestData } from "@/types/message"; -import type { SQL } from "drizzle-orm"; -import { PgDialect } from "drizzle-orm/pg-core"; import { afterEach, describe, expect, it, vi } from "vitest"; function installSyncBoundaries(insertedRows: readonly Record[] = []) { @@ -29,16 +27,9 @@ function installSyncBoundaries(insertedRows: readonly Record[] function installAsyncRoutingTraceBoundaries() { const controlUpdate = vi.fn(); - const writerUpdateWhere = vi.fn(async (_condition: unknown) => []); - const writerUpdateSet = vi.fn((_values: Record) => ({ - where: writerUpdateWhere, - })); - const writerUpdate = vi.fn((_table: unknown) => ({ set: writerUpdateSet })); - const getMessageWriterDb = vi.fn(() => ({ - update: writerUpdate, - execute: vi.fn(), - })); + const getMessageWriterDb = vi.fn(); const enqueueMessageRequestUpdate = vi.fn(); + const enqueueMessageRequestPostTerminalRoutingTraceDurably = vi.fn(async () => true); const loggerWarn = vi.fn(); vi.doMock("@/drizzle/db", () => ({ @@ -57,6 +48,7 @@ function installAsyncRoutingTraceBoundaries() { isDevelopment: vi.fn(() => false), })); vi.doMock("@/repository/message-write-buffer", () => ({ + enqueueMessageRequestPostTerminalRoutingTraceDurably, enqueueMessageRequestUpdate, enqueueMessageRequestUpdateDurably: vi.fn(), })); @@ -71,12 +63,10 @@ function installAsyncRoutingTraceBoundaries() { return { controlUpdate, + enqueueMessageRequestPostTerminalRoutingTraceDurably, enqueueMessageRequestUpdate, getMessageWriterDb, loggerWarn, - writerUpdate, - writerUpdateSet, - writerUpdateWhere, }; } @@ -264,7 +254,7 @@ describe("message terminal write APIs", () => { expect(updateWhere).toHaveBeenCalledTimes(1); }); - it("patches an async finalized routing trace directly through the writer lane", async () => { + it("patches an async finalized routing trace through acknowledged post-terminal metadata", async () => { vi.resetModules(); const boundaries = installAsyncRoutingTraceBoundaries(); const { updateMessageRequestRoutingTrace } = await import("@/repository/message"); @@ -290,54 +280,20 @@ describe("message terminal write APIs", () => { expect(boundaries.enqueueMessageRequestUpdate).not.toHaveBeenCalled(); expect(boundaries.controlUpdate).not.toHaveBeenCalled(); - expect(boundaries.getMessageWriterDb).toHaveBeenCalledOnce(); - expect(boundaries.writerUpdate).toHaveBeenCalledOnce(); - expect(boundaries.writerUpdateSet).toHaveBeenCalledWith({ - routingTrace, - updatedAt: expect.any(Date), - }); - expect(boundaries.writerUpdateSet.mock.calls[0]?.[0]).not.toHaveProperty("statusCode"); - expect(boundaries.writerUpdateSet.mock.calls[0]?.[0]).not.toHaveProperty("costUsd"); - - const where = boundaries.writerUpdateWhere.mock.calls[0]?.[0] as SQL; - const query = new PgDialect().sqlToQuery(where); - expect(query.sql).toContain('"message_request"."id" = $1'); - expect(query.sql).toContain('"message_request"."deleted_at" is null'); - expect(query.sql).not.toContain("status_code"); - }); - - it("retries a transient async routing trace write before succeeding", async () => { - vi.useFakeTimers(); - vi.resetModules(); - const boundaries = installAsyncRoutingTraceBoundaries(); - boundaries.writerUpdateWhere - .mockRejectedValueOnce(new Error("writer temporarily unavailable")) - .mockRejectedValueOnce(new Error("writer still unavailable")) - .mockResolvedValueOnce([]); - const { updateMessageRequestRoutingTrace } = await import("@/repository/message"); - const routingTrace = { - version: 1 as const, - mode: "legacy_serial" as const, - startedAt: 2_000, - updatedAt: 2_100, - discoveryEnabled: false, - eligible: false, - events: [], - }; - - const persistence = updateMessageRequestRoutingTrace(708, routingTrace); - await vi.runAllTimersAsync(); - await expect(persistence).resolves.toBeUndefined(); - - expect(boundaries.writerUpdateWhere).toHaveBeenCalledTimes(3); + expect(boundaries.getMessageWriterDb).not.toHaveBeenCalled(); + expect(boundaries.enqueueMessageRequestPostTerminalRoutingTraceDurably).toHaveBeenCalledWith( + 707, + routingTrace + ); expect(boundaries.loggerWarn).not.toHaveBeenCalled(); }); - it("keeps exhausted async routing trace persistence best-effort and logs once", async () => { - vi.useFakeTimers(); + it("keeps rejected async routing trace persistence best-effort and logs once", async () => { vi.resetModules(); const boundaries = installAsyncRoutingTraceBoundaries(); - boundaries.writerUpdateWhere.mockRejectedValue(new Error("writer unavailable")); + boundaries.enqueueMessageRequestPostTerminalRoutingTraceDurably.mockRejectedValue( + new Error("durable writer unavailable") + ); const { updateMessageRequestRoutingTrace } = await import("@/repository/message"); const routingTrace = { version: 1 as const, @@ -349,19 +305,33 @@ describe("message terminal write APIs", () => { events: [], }; - const persistence = updateMessageRequestRoutingTrace(709, routingTrace); - await vi.runAllTimersAsync(); - await expect(persistence).resolves.toBeUndefined(); + await expect(updateMessageRequestRoutingTrace(709, routingTrace)).resolves.toBeUndefined(); - expect(boundaries.writerUpdateWhere).toHaveBeenCalledTimes(3); + expect(boundaries.enqueueMessageRequestPostTerminalRoutingTraceDurably).toHaveBeenCalledOnce(); expect(boundaries.loggerWarn).toHaveBeenCalledOnce(); expect(boundaries.loggerWarn).toHaveBeenCalledWith( "[MessageRequest] Failed to patch finalized routing trace", { requestId: 709, - attempts: 3, - error: "writer unavailable", + error: "durable writer unavailable", } ); }); + + it("logs and skips an invalid routing trace without persisting raw trace data", async () => { + vi.resetModules(); + const boundaries = installAsyncRoutingTraceBoundaries(); + const { updateMessageRequestRoutingTrace } = await import("@/repository/message"); + + await updateMessageRequestRoutingTrace(710, { version: 2 } as unknown as Parameters< + typeof updateMessageRequestRoutingTrace + >[1]); + + expect(boundaries.enqueueMessageRequestPostTerminalRoutingTraceDurably).not.toHaveBeenCalled(); + expect(boundaries.controlUpdate).not.toHaveBeenCalled(); + expect(boundaries.loggerWarn).toHaveBeenCalledWith( + "[MessageRequest] Skipped patching invalid routing trace", + { requestId: 710 } + ); + }); }); diff --git a/tests/unit/repository/message-write-buffer.test.ts b/tests/unit/repository/message-write-buffer.test.ts index 8602dc78e..09832d09d 100644 --- a/tests/unit/repository/message-write-buffer.test.ts +++ b/tests/unit/repository/message-write-buffer.test.ts @@ -47,6 +47,26 @@ function createDeferred() { return { promise, resolve, reject }; } +function createRoutingTrace(at: number) { + return { + version: 1 as const, + mode: "discovery" as const, + startedAt: at - 100, + updatedAt: at, + discoveryEnabled: true, + eligible: true, + events: [ + { + type: "binding_finalized" as const, + at, + elapsedMs: 100, + bindingAction: "create" as const, + outcome: "updated", + }, + ], + }; +} + describe("message_request 异步批量写入", () => { const envKeys = [ "NODE_ENV", @@ -240,6 +260,313 @@ describe("message_request 异步批量写入", () => { await stopMessageRequestWriteBuffer(); }); + it("post-terminal routing trace uses RETURNING ACK without the terminal status fence", async () => { + process.env.MESSAGE_REQUEST_WRITE_MODE = "async"; + + const { + enqueueMessageRequestPostTerminalRoutingTraceDurably, + flushMessageRequestWriteBuffer, + stopMessageRequestWriteBuffer, + } = await import("@/repository/message-write-buffer"); + const routingTrace = createRoutingTrace(1_100); + const metadata = enqueueMessageRequestPostTerminalRoutingTraceDurably(52_001, routingTrace); + + await flushMessageRequestWriteBuffer(); + await expect(metadata).resolves.toBe(true); + await stopMessageRequestWriteBuffer(); + + const built = toSqlText(executeMock.mock.calls[0]?.[0]); + expect(built.sql).toContain("RETURNING id"); + expect(built.sql).toContain("routing_trace"); + expect(built.sql).not.toContain("status_code IS NULL"); + expect(built.sql).not.toContain("duration_ms"); + expect( + built.params.some( + (value) => typeof value === "string" && value.includes('"binding_finalized"') + ) + ).toBe(true); + }); + + it("keeps post-terminal routing trace pending beyond three transient DB failures", async () => { + process.env.MESSAGE_REQUEST_WRITE_MODE = "async"; + executeMock + .mockRejectedValueOnce(new Error("writer unavailable 1")) + .mockRejectedValueOnce(new Error("writer unavailable 2")) + .mockRejectedValueOnce(new Error("writer unavailable 3")); + + const { + enqueueMessageRequestPostTerminalRoutingTraceDurably, + flushMessageRequestWriteBuffer, + stopMessageRequestWriteBuffer, + } = await import("@/repository/message-write-buffer"); + const metadata = enqueueMessageRequestPostTerminalRoutingTraceDurably( + 52_005, + createRoutingTrace(1_500) + ); + let settled = false; + void metadata.finally(() => { + settled = true; + }); + + await flushMessageRequestWriteBuffer(); + await flushMessageRequestWriteBuffer(); + await flushMessageRequestWriteBuffer(); + expect(settled).toBe(false); + + await flushMessageRequestWriteBuffer(); + await expect(metadata).resolves.toBe(true); + expect(executeMock).toHaveBeenCalledTimes(4); + await stopMessageRequestWriteBuffer(); + }); + + it("waits for the same id terminal ACK before enqueuing post-terminal metadata", async () => { + process.env.MESSAGE_REQUEST_WRITE_MODE = "async"; + const deferred = createDeferred(); + executeMock.mockImplementationOnce(async () => deferred.promise); + + const { + enqueueMessageRequestUpdateDurably, + enqueueMessageRequestPostTerminalRoutingTraceDurably, + flushMessageRequestWriteBuffer, + stopMessageRequestWriteBuffer, + } = await import("@/repository/message-write-buffer"); + const terminal = enqueueMessageRequestUpdateDurably(52_002, { statusCode: 200 }); + const terminalFlush = flushMessageRequestWriteBuffer(); + const metadata = enqueueMessageRequestPostTerminalRoutingTraceDurably( + 52_002, + createRoutingTrace(2_100) + ); + let metadataSettled = false; + void metadata.finally(() => { + metadataSettled = true; + }); + await new Promise((resolve) => setImmediate(resolve)); + expect(metadataSettled).toBe(false); + + deferred.resolve([{ id: 52_002 }]); + await terminalFlush; + await terminal; + await flushMessageRequestWriteBuffer(); + await expect(metadata).resolves.toBe(true); + await stopMessageRequestWriteBuffer(); + + expect(executeMock).toHaveBeenCalledTimes(2); + const terminalSql = toSqlText(executeMock.mock.calls[0]?.[0]); + const metadataSql = toSqlText(executeMock.mock.calls[1]?.[0]); + expect(terminalSql.sql).toContain('"status_code" IS NULL'); + expect(metadataSql.sql).not.toContain('"status_code" IS NULL'); + expect(metadataSql.sql).toContain("routing_trace"); + }); + + it("does not merge ordinary updates into an in-flight post-terminal metadata ACK", async () => { + process.env.MESSAGE_REQUEST_WRITE_MODE = "async"; + const releaseMetadata = createDeferred(); + executeMock.mockImplementationOnce(async () => releaseMetadata.promise); + + const { + enqueueMessageRequestUpdate, + enqueueMessageRequestPostTerminalRoutingTraceDurably, + flushMessageRequestWriteBuffer, + stopMessageRequestWriteBuffer, + } = await import("@/repository/message-write-buffer"); + const metadata = enqueueMessageRequestPostTerminalRoutingTraceDurably( + 52_004, + createRoutingTrace(4_100) + ); + const metadataFlush = flushMessageRequestWriteBuffer(); + enqueueMessageRequestUpdate(52_004, { durationMs: 654 }); + + releaseMetadata.resolve([{ id: 52_004 }]); + await metadataFlush; + await expect(metadata).resolves.toBe(true); + await flushMessageRequestWriteBuffer(); + await stopMessageRequestWriteBuffer(); + + expect(executeMock).toHaveBeenCalledTimes(2); + const metadataSql = toSqlText(executeMock.mock.calls[0]?.[0]); + const ordinarySql = toSqlText(executeMock.mock.calls[1]?.[0]); + expect(metadataSql.sql).toContain("routing_trace"); + expect(metadataSql.sql).not.toContain("duration_ms"); + expect(ordinarySql.sql).toContain("duration_ms"); + expect(ordinarySql.sql).not.toContain("routing_trace"); + }); + + it("keeps a failed in-flight ordinary requeue isolated from a newer metadata ACK", async () => { + process.env.MESSAGE_REQUEST_WRITE_MODE = "async"; + const releaseOrdinary = createDeferred(); + executeMock.mockImplementationOnce(async () => { + await releaseOrdinary.promise; + throw new Error("ordinary write failed"); + }); + + const { + enqueueMessageRequestUpdate, + enqueueMessageRequestPostTerminalRoutingTraceDurably, + flushMessageRequestWriteBuffer, + stopMessageRequestWriteBuffer, + } = await import("@/repository/message-write-buffer"); + enqueueMessageRequestUpdate(52_006, { durationMs: 777 }); + const flush = flushMessageRequestWriteBuffer(); + const metadata = enqueueMessageRequestPostTerminalRoutingTraceDurably( + 52_006, + createRoutingTrace(6_100) + ); + + releaseOrdinary.resolve(); + await flush; + await expect(metadata).resolves.toBe(true); + await stopMessageRequestWriteBuffer(); + + expect(executeMock).toHaveBeenCalledTimes(3); + const failedOrdinarySql = toSqlText(executeMock.mock.calls[0]?.[0]); + const metadataSql = toSqlText(executeMock.mock.calls[1]?.[0]); + const retriedOrdinarySql = toSqlText(executeMock.mock.calls[2]?.[0]); + expect(failedOrdinarySql.sql).toContain("duration_ms"); + expect(metadataSql.sql).toContain("routing_trace"); + expect(metadataSql.sql).not.toContain("duration_ms"); + expect(retriedOrdinarySql.sql).toContain("duration_ms"); + expect(retriedOrdinarySql.sql).not.toContain("routing_trace"); + }); + + it("coalesces duplicate metadata callers and counts unique tasks against maxPending", async () => { + process.env.MESSAGE_REQUEST_WRITE_MODE = "async"; + process.env.MESSAGE_REQUEST_ASYNC_MAX_PENDING = "100"; + const trace = createRoutingTrace(7_100); + + const { + enqueueMessageRequestPostTerminalRoutingTraceDurably, + flushMessageRequestWriteBuffer, + stopMessageRequestWriteBuffer, + } = await import("@/repository/message-write-buffer"); + const first = enqueueMessageRequestPostTerminalRoutingTraceDurably(52_007, trace); + const duplicate = enqueueMessageRequestPostTerminalRoutingTraceDurably(52_007, trace); + const admitted = [first]; + for (let index = 1; index < 100; index++) { + admitted.push( + enqueueMessageRequestPostTerminalRoutingTraceDurably( + 52_007 + index, + createRoutingTrace(7_100 + index) + ) + ); + } + const overflow = enqueueMessageRequestPostTerminalRoutingTraceDurably( + 53_000, + createRoutingTrace(7_300) + ); + + expect(duplicate).toBe(first); + await expect(overflow).rejects.toThrow("durable message_request queue is full"); + await flushMessageRequestWriteBuffer(); + await expect(Promise.all(admitted)).resolves.toHaveLength(100); + expect(executeMock).toHaveBeenCalledTimes(1); + await stopMessageRequestWriteBuffer(); + }); + + it("flushes an existing ordinary patch before isolating post-terminal metadata", async () => { + process.env.MESSAGE_REQUEST_WRITE_MODE = "async"; + + const { + enqueueMessageRequestUpdate, + enqueueMessageRequestPostTerminalRoutingTraceDurably, + flushMessageRequestWriteBuffer, + stopMessageRequestWriteBuffer, + } = await import("@/repository/message-write-buffer"); + enqueueMessageRequestUpdate(52_003, { durationMs: 321 }); + const metadata = enqueueMessageRequestPostTerminalRoutingTraceDurably( + 52_003, + createRoutingTrace(3_100) + ); + + await flushMessageRequestWriteBuffer(); + await flushMessageRequestWriteBuffer(); + await expect(metadata).resolves.toBe(true); + await stopMessageRequestWriteBuffer(); + + expect(executeMock).toHaveBeenCalledTimes(2); + const ordinarySql = toSqlText(executeMock.mock.calls[0]?.[0]); + const metadataSql = toSqlText(executeMock.mock.calls[1]?.[0]); + expect(ordinarySql.sql).toContain("duration_ms"); + expect(ordinarySql.sql).not.toContain("routing_trace"); + expect(metadataSql.sql).toContain("routing_trace"); + expect(metadataSql.sql).not.toContain("duration_ms"); + }); + + it("stop waits for a callback's late post-terminal metadata ACK", async () => { + process.env.MESSAGE_REQUEST_WRITE_MODE = "async"; + + const { + enqueueMessageRequestUpdateDurably, + enqueueMessageRequestPostTerminalRoutingTraceDurably, + stopMessageRequestWriteBuffer, + } = await import("@/repository/message-write-buffer"); + let metadataResult: unknown; + const terminal = enqueueMessageRequestUpdateDurably( + 52_004, + { statusCode: 200 }, + { + onCommitted: async () => { + metadataResult = await enqueueMessageRequestPostTerminalRoutingTraceDurably( + 52_004, + createRoutingTrace(4_100) + ); + }, + } + ); + + await stopMessageRequestWriteBuffer(); + await expect(terminal).resolves.toBe(true); + expect(metadataResult).toBe(true); + + expect(executeMock).toHaveBeenCalledTimes(2); + const metadataSql = toSqlText(executeMock.mock.calls[1]?.[0]); + expect(metadataSql.sql).toContain("RETURNING id"); + expect(metadataSql.sql).toContain("routing_trace"); + expect(metadataSql.sql).not.toContain("status_code IS NULL"); + }); + + it("stop bounds a failing late metadata write to the shutdown flush budget", async () => { + process.env.MESSAGE_REQUEST_WRITE_MODE = "async"; + executeMock + .mockImplementationOnce(async (query) => successfulRowsForQuery(query)) + .mockRejectedValue(new Error("writer unavailable during shutdown")); + + const { + enqueueMessageRequestUpdateDurably, + enqueueMessageRequestPostTerminalRoutingTraceDurably, + stopMessageRequestWriteBuffer, + } = await import("@/repository/message-write-buffer"); + let metadata!: Promise; + const terminal = enqueueMessageRequestUpdateDurably( + 52_009, + { statusCode: 200 }, + { + onCommitted: () => { + metadata = enqueueMessageRequestPostTerminalRoutingTraceDurably( + 52_009, + createRoutingTrace(9_100) + ); + return metadata; + }, + } + ); + + await expect(stopMessageRequestWriteBuffer()).rejects.toThrow( + "message_request writer shutdown persistence failed" + ); + await expect(terminal).resolves.toBe(true); + await expect(metadata).rejects.toThrow( + "post-terminal metadata did not persist during writer shutdown" + ); + expect(executeMock).toHaveBeenCalledTimes(3); + expect(loggerErrorMock).toHaveBeenCalledWith( + "[MessageRequestWriteBuffer] Durable commit callback failed", + expect.objectContaining({ + error: "post-terminal metadata did not persist during writer shutdown", + messageRequestId: 52_009, + }) + ); + }); + it("多个 durable 终态应由同一次 batch flush 共同确认", async () => { process.env.MESSAGE_REQUEST_WRITE_MODE = "async"; From 01277fc26f4756ced7e395ec7ec867a36166c114 Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Tue, 21 Jul 2026 08:57:21 -0400 Subject: [PATCH 06/12] fix(observability): make final routing trace recoverable --- src/app/v1/_lib/proxy/session.ts | 9 +- src/instrumentation.ts | 18 + src/lib/lifecycle/shutdown.ts | 32 +- src/repository/message-write-buffer.ts | 91 +++- src/repository/message.ts | 56 ++- src/repository/routing-trace-outbox.ts | 424 ++++++++++++++++++ src/repository/routing-trace-persistence.ts | 41 ++ tests/unit/lib/shutdown.test.ts | 54 +++ tests/unit/proxy/routing-trace.test.ts | 29 ++ .../message-terminal-write-apis.test.ts | 162 ++++++- .../repository/message-write-buffer.test.ts | 57 +++ .../repository/routing-trace-outbox.test.ts | 318 +++++++++++++ 12 files changed, 1255 insertions(+), 36 deletions(-) create mode 100644 src/repository/routing-trace-outbox.ts create mode 100644 src/repository/routing-trace-persistence.ts create mode 100644 tests/unit/repository/routing-trace-outbox.test.ts diff --git a/src/app/v1/_lib/proxy/session.ts b/src/app/v1/_lib/proxy/session.ts index d0472c940..a91a2c25c 100644 --- a/src/app/v1/_lib/proxy/session.ts +++ b/src/app/v1/_lib/proxy/session.ts @@ -814,6 +814,11 @@ export class ProxySession { this.scheduleLiveObservabilityFlush(); } + private advanceRoutingTraceRevision(observedAt: number): void { + if (!this.routingTrace) return; + this.routingTrace.updatedAt = Math.max(observedAt, this.routingTrace.updatedAt + 1); + } + initializeRoutingTrace(options: { mode: RoutingTraceMode; discoveryEnabled: boolean; @@ -882,7 +887,7 @@ export class ProxySession { } } if (!changed) return; - this.routingTrace.updatedAt = at; + this.advanceRoutingTraceRevision(at); this.persistLiveRoutingTrace(); } @@ -933,7 +938,7 @@ export class ProxySession { terminalEvent.elapsedMs = Math.max(0, now - this.routingTrace.startedAt); terminalEvent.outcome = resolvedOutcome; terminalEvent.statusCode = statusCode; - this.routingTrace.updatedAt = now; + this.advanceRoutingTraceRevision(now); this.persistLiveRoutingTrace(); } return this.getRoutingTrace(); diff --git a/src/instrumentation.ts b/src/instrumentation.ts index 3da834224..2827dcd57 100644 --- a/src/instrumentation.ts +++ b/src/instrumentation.ts @@ -233,6 +233,21 @@ async function startCloudPriceSyncScheduler(): Promise { } } +async function startRoutingTraceOutboxRecovery(): Promise { + if (!process.env.REDIS_URL) return; + try { + const { startRoutingTraceOutboxReplayScheduler } = await import( + "@/repository/routing-trace-outbox" + ); + await startRoutingTraceOutboxReplayScheduler(); + logger.info("[Instrumentation] Routing trace outbox recovery started"); + } catch (error) { + logger.warn("[Instrumentation] Routing trace outbox recovery failed to start", { + error: error instanceof Error ? error.message : String(error), + }); + } +} + /** * 多实例：订阅 API Key 变更广播，触发本机 Vacuum Filter 失效并重建。 * @@ -358,6 +373,8 @@ export async function register() { }); } + await startRoutingTraceOutboxRecovery(); + // Ledger backfill: fire-and-forget after migration (non-blocking, idempotent) Promise.all([import("@/lib/async-task-manager"), import("@/lib/ledger-backfill")]) .then(([{ AsyncTaskManager }, { backfillUsageLedger }]) => { @@ -528,6 +545,7 @@ export async function register() { const isConnected = await checkDatabaseConnection(); if (isConnected) { await runMigrations(); + await startRoutingTraceOutboxRecovery(); // Ledger backfill: fire-and-forget after migration (non-blocking, idempotent) Promise.all([import("@/lib/async-task-manager"), import("@/lib/ledger-backfill")]) diff --git a/src/lib/lifecycle/shutdown.ts b/src/lib/lifecycle/shutdown.ts index 42aa4f800..23443ee95 100644 --- a/src/lib/lifecycle/shutdown.ts +++ b/src/lib/lifecycle/shutdown.ts @@ -100,9 +100,20 @@ export async function runApplicationCleanup( (async () => { const { stopCacheCleanup } = await import("@/lib/cache/session-cache"); stopCacheCleanup(); + const stopRoutingTraceOutboxReplayScheduler = ( + globalThis as typeof globalThis & { + __CCH_STOP_ROUTING_TRACE_OUTBOX__?: (options?: { + wait?: boolean; + maxWaitMs?: number; + }) => Promise; + } + ).__CCH_STOP_ROUTING_TRACE_OUTBOX__; + // Stop future ticks immediately, but do not let a slow metadata replay + // block async-task and message-writer quiescence below. + await stopRoutingTraceOutboxReplayScheduler?.({ wait: false }); })(), stepMs, - "stopCacheCleanup" + "stopLocalSchedulers" ); // 2. 端点探测调度器 @@ -199,6 +210,25 @@ export async function runApplicationCleanup( writerQuiescencePending = false; } + // The writer has now settled. Give the already-running outbox cycle a + // bounded chance to finish while DB/Redis are still open; a retained Hash + // entry is the recovery path if the cycle is still blocked. + await awaitQuiescenceBestEffort( + (async () => { + const stopRoutingTraceOutboxReplayScheduler = ( + globalThis as typeof globalThis & { + __CCH_STOP_ROUTING_TRACE_OUTBOX__?: (options?: { + wait?: boolean; + maxWaitMs?: number; + }) => Promise; + } + ).__CCH_STOP_ROUTING_TRACE_OUTBOX__; + await stopRoutingTraceOutboxReplayScheduler?.({ wait: true, maxWaitMs: stepMs }); + })(), + stepMs, + "stopRoutingTraceOutboxReplay" + ); + // 8. writer flush 完成后再关闭数据库 pool。pool close 也是 critical barrier， // 单步 deadline 只能告警，不能让底层 client.end() 脱离 shutdown 生命周期。 const dbWarningTimer = setTimeout(() => { diff --git a/src/repository/message-write-buffer.ts b/src/repository/message-write-buffer.ts index c5199893a..602ce316e 100644 --- a/src/repository/message-write-buffer.ts +++ b/src/repository/message-write-buffer.ts @@ -9,6 +9,7 @@ import { logger } from "@/lib/logger"; import type { StoredCostBreakdown } from "@/types/cost-breakdown"; import type { CreateMessageRequestData } from "@/types/message"; import { normalizeRoutingTrace, type RoutingTraceV1 } from "@/types/routing-trace"; +import { buildMonotonicRoutingTraceAssignments } from "./routing-trace-persistence"; export type MessageRequestUpdatePatch = { durationMs?: number; @@ -79,6 +80,12 @@ type MessageRequestUpdateBatchRecord = MessageRequestUpdateRecord & { durableAcknowledgement?: DurableAcknowledgement; }; +type PostTerminalMetadataTask = { + promise: Promise; + routingTraceUpdatedAt: number; + routingTracePayload: string; +}; + type WriterConfig = { flushIntervalMs: number; batchSize: number; @@ -307,13 +314,18 @@ function takeBatch( export function buildBatchUpdateSql( updates: MessageRequestUpdateRecord[], - options: { returnUpdatedIds?: boolean; fencedDurableIds?: readonly number[] } = {} + options: { + returnUpdatedIds?: boolean; + fencedDurableIds?: readonly number[]; + monotonicRoutingTraceIds?: readonly number[]; + } = {} ): SQL | null { if (updates.length === 0) { return null; } const ids = updates.map((u) => u.id); + const monotonicRoutingTraceIds = new Set(options.monotonicRoutingTraceIds ?? []); const setClauses: SQL[] = []; for (const [key, columnName] of Object.entries(COLUMN_MAP) as Array< @@ -336,8 +348,24 @@ export function buildBatchUpdateSql( cases.push(sql`WHEN ${update.id} THEN NULL`); continue; } - const json = JSON.stringify(key === "routingTrace" ? normalizeRoutingTrace(value) : value); - cases.push(sql`WHEN ${update.id} THEN ${json}::jsonb`); + if (key === "routingTrace") { + const normalizedTrace = normalizeRoutingTrace(value); + if (!normalizedTrace) { + cases.push(sql`WHEN ${update.id} THEN NULL`); + continue; + } + if (!monotonicRoutingTraceIds.has(update.id)) { + cases.push(sql`WHEN ${update.id} THEN ${JSON.stringify(normalizedTrace)}::jsonb`); + continue; + } + const assignments = buildMonotonicRoutingTraceAssignments(normalizedTrace, { + routingTrace: sql`${sql.identifier("routing_trace")}`, + updatedAt: sql`${sql.identifier("updated_at")}`, + }); + cases.push(sql`WHEN ${update.id} THEN ${assignments.routingTrace}`); + continue; + } + cases.push(sql`WHEN ${update.id} THEN ${JSON.stringify(value)}::jsonb`); continue; } @@ -363,8 +391,24 @@ export function buildBatchUpdateSql( return null; } - // 所有更新统一刷新 updated_at - setClauses.push(sql`${sql.identifier("updated_at")} = NOW()`); + if (monotonicRoutingTraceIds.size > 0) { + const cases: SQL[] = []; + for (const update of updates) { + if (!monotonicRoutingTraceIds.has(update.id) || !update.patch.routingTrace) continue; + const assignments = buildMonotonicRoutingTraceAssignments(update.patch.routingTrace, { + routingTrace: sql`${sql.identifier("routing_trace")}`, + updatedAt: sql`${sql.identifier("updated_at")}`, + }); + cases.push(sql`WHEN ${update.id} THEN ${assignments.updatedAt}`); + } + setClauses.push( + cases.length > 0 + ? sql`${sql.identifier("updated_at")} = CASE id ${sql.join(cases, sql` `)} ELSE NOW() END` + : sql`${sql.identifier("updated_at")} = NOW()` + ); + } else { + setClauses.push(sql`${sql.identifier("updated_at")} = NOW()`); + } const idList = sql.join( ids.map((id) => sql`${id}`), @@ -429,7 +473,7 @@ class MessageRequestWriteBuffer { private readonly config: WriterConfig; private readonly pending = new Map(); private readonly deferredOrdinary = new Map(); - private readonly postTerminalMetadataTasks = new Map>(); + private readonly postTerminalMetadataTasks = new Map(); private readonly evictableIndex = new EvictablePendingIndex(); private readonly durableAcknowledgements = new Map(); private flushTimer: NodeJS.Timeout | null = null; @@ -521,11 +565,22 @@ class MessageRequestWriteBuffer { new Error("post-terminal metadata updates may only contain routingTrace") ); } + const normalizedTrace = normalizeRoutingTrace(patch.routingTrace); + if (!normalizedTrace) { + return Promise.reject(new Error("post-terminal routing trace is invalid")); + } + const routingTracePayload = JSON.stringify(normalizedTrace); const existingTask = this.postTerminalMetadataTasks.get(id); if (existingTask) { - // Binding finalization is once-guarded upstream. Coalesce accidental - // duplicate callers so they cannot allocate unbounded waiters/timers. - return existingTask; + // Exact duplicates may share the same ACK. A different revision remains + // in the Redis outbox and must not be acknowledged as if this SQL wrote it. + if ( + existingTask.routingTraceUpdatedAt === normalizedTrace.updatedAt && + existingTask.routingTracePayload === routingTracePayload + ) { + return existingTask.promise; + } + return existingTask.promise.then(() => false); } if ( this.stopping && @@ -537,14 +592,18 @@ class MessageRequestWriteBuffer { return Promise.reject(new Error("durable message_request queue is full")); } - let task: Promise; - task = this.persistPostTerminalMetadataDurably(id, patch, options).finally(() => { + const task: PostTerminalMetadataTask = { + promise: Promise.resolve(false), + routingTraceUpdatedAt: normalizedTrace.updatedAt, + routingTracePayload, + }; + task.promise = this.persistPostTerminalMetadataDurably(id, patch, options).finally(() => { if (this.postTerminalMetadataTasks.get(id) === task) { this.postTerminalMetadataTasks.delete(id); } }); this.postTerminalMetadataTasks.set(id, task); - return task; + return task.promise; } private async persistPostTerminalMetadataDurably( @@ -916,9 +975,13 @@ class MessageRequestWriteBuffer { const fencedDurableIds = batch.flatMap((item) => item.durableAcknowledgement?.writeScope === "terminal" ? [item.id] : [] ); + const monotonicRoutingTraceIds = batch.flatMap((item) => + item.durableAcknowledgement?.writeScope === "post-terminal-metadata" ? [item.id] : [] + ); const query = buildBatchUpdateSql(batch, { returnUpdatedIds: requiresUpdatedIds, fencedDurableIds, + monotonicRoutingTraceIds, }); if (!query) { for (const item of batch) { @@ -1040,7 +1103,7 @@ class MessageRequestWriteBuffer { ) { await Promise.allSettled([ ...this.commitCallbacksInFlight, - ...this.postTerminalMetadataTasks.values(), + ...Array.from(this.postTerminalMetadataTasks.values(), (task) => task.promise), ]); } @@ -1061,7 +1124,7 @@ class MessageRequestWriteBuffer { while (this.commitCallbacksInFlight.size > 0 || this.postTerminalMetadataTasks.size > 0) { await Promise.allSettled([ ...this.commitCallbacksInFlight, - ...this.postTerminalMetadataTasks.values(), + ...Array.from(this.postTerminalMetadataTasks.values(), (task) => task.promise), ]); } this.clearOverflowLogTimer(); diff --git a/src/repository/message.ts b/src/repository/message.ts index 1ec782675..25483e3cc 100644 --- a/src/repository/message.ts +++ b/src/repository/message.ts @@ -25,6 +25,13 @@ import { enqueueMessageRequestUpdateDurably, type MessageRequestUpdatePatch, } from "./message-write-buffer"; +import { + acknowledgeRoutingTraceOutbox, + persistRoutingTraceMonotonically, + stageRoutingTraceOutbox, +} from "./routing-trace-outbox"; + +const POST_TERMINAL_ROUTING_TRACE_ACK_TIMEOUT_MS = 3_000; type PublicStatusRequestSeed = { createdAt: Date; @@ -616,9 +623,10 @@ export async function updateMessageRequestDetails( } /** - * Best-effort routing trace patch for work that completes after the request's - * terminal row has been committed (for example, Sticky binding finalization). - * This intentionally bypasses terminal ownership and public-status rollups. + * Routing trace patch for work that completes after the request's terminal row + * has committed. A Redis outbox is staged first so a shutdown-time database + * outage can be replayed after restart without touching terminal ownership, + * billing, or public-status rollups. */ export async function updateMessageRequestRoutingTrace( id: number, @@ -632,29 +640,59 @@ export async function updateMessageRequestRoutingTrace( return; } + const outboxReceipt = await stageRoutingTraceOutbox(id, normalized); + let persisted = false; + if (getEnvConfig().MESSAGE_REQUEST_WRITE_MODE === "async") { + let persistenceError: unknown = null; try { - await enqueueMessageRequestPostTerminalRoutingTraceDurably(id, normalized); + persisted = await enqueueMessageRequestPostTerminalRoutingTraceDurably(id, normalized, { + timeoutMs: POST_TERMINAL_ROUTING_TRACE_ACK_TIMEOUT_MS, + }); } catch (error) { + persistenceError = error; + } + + // A different revision may be coalesced behind an older in-flight writer + // task. Normally its outbox receipt is the recovery path; if Redis staging + // was unavailable, make one monotonic direct attempt instead of dropping it. + if (!persisted && !outboxReceipt) { + try { + await persistRoutingTraceMonotonically(id, normalized); + persisted = true; + persistenceError = null; + } catch (error) { + persistenceError = error; + } + } + + if (!persisted && persistenceError) { logger.warn("[MessageRequest] Failed to patch finalized routing trace", { requestId: id, - error: error instanceof Error ? error.message : String(error), + recoverable: outboxReceipt !== null, + error: + persistenceError instanceof Error ? persistenceError.message : String(persistenceError), }); } + if (persisted && outboxReceipt) { + await acknowledgeRoutingTraceOutbox(outboxReceipt); + } return; } try { - await db - .update(messageRequest) - .set({ routingTrace: normalized, updatedAt: new Date() }) - .where(and(eq(messageRequest.id, id), isNull(messageRequest.deletedAt))); + await persistRoutingTraceMonotonically(id, normalized); + persisted = true; } catch (error) { logger.warn("[MessageRequest] Failed to patch finalized routing trace", { requestId: id, + recoverable: outboxReceipt !== null, error: error instanceof Error ? error.message : String(error), }); } + if (persisted && outboxReceipt) { + await acknowledgeRoutingTraceOutbox(outboxReceipt); + } } export async function updateMessageRequestDetailsIfUnfinalized( diff --git a/src/repository/routing-trace-outbox.ts b/src/repository/routing-trace-outbox.ts new file mode 100644 index 000000000..4afde2fb4 --- /dev/null +++ b/src/repository/routing-trace-outbox.ts @@ -0,0 +1,424 @@ +import "server-only"; + +import { and, eq, isNull, sql } from "drizzle-orm"; +import type Redis from "ioredis"; +import { getMessageWriterDb } from "@/drizzle/db"; +import { messageRequest } from "@/drizzle/schema"; +import { logger } from "@/lib/logger"; +import { getRedisClient } from "@/lib/redis/client"; +import { normalizeRoutingTrace, type RoutingTraceV1 } from "@/types/routing-trace"; +import { buildMonotonicRoutingTraceAssignments } from "./routing-trace-persistence"; + +const ROUTING_TRACE_OUTBOX_KEY = "cch:routing-trace-outbox:v1"; +const DEFAULT_REPLAY_LIMIT = 100; +const REPLAY_INTERVAL_MS = 30_000; +const REDIS_READY_WAIT_MS = 500; +const REDIS_OPERATION_TIMEOUT_MS = 1_000; +const BACKLOG_WARN_THRESHOLD = 1_000; +const BACKLOG_ERROR_THRESHOLD = 10_000; +const BACKLOG_LOG_INTERVAL_MS = 5 * 60_000; + +let lastBacklogLogAt = 0; + +const STAGE_IF_NOT_OLDER_LUA = ` +local current = redis.call('HGET', KEYS[1], ARGV[1]) +if current then + local ok, decoded = pcall(cjson.decode, current) + local current_revision = nil + if ok and decoded then + current_revision = tonumber(decoded.traceUpdatedAt) + end + local incoming_revision = tonumber(ARGV[2]) + if current_revision and incoming_revision and current_revision > incoming_revision then + return 0 + end + if current_revision and incoming_revision and current_revision == incoming_revision and current ~= ARGV[3] then + return 0 + end +end +redis.call('HSET', KEYS[1], ARGV[1], ARGV[3]) +return 1`; + +const DELETE_IF_UNCHANGED_LUA = ` +local current = redis.call('HGET', KEYS[1], ARGV[1]) +if current == ARGV[2] then + return redis.call('HDEL', KEYS[1], ARGV[1]) +end +return 0`; + +type RoutingTraceOutboxEntry = { + version: 1; + requestId: number; + traceUpdatedAt: number; + routingTrace: RoutingTraceV1; +}; + +export type RoutingTraceOutboxReceipt = { + field: string; + payload: string; +}; + +export type RoutingTraceOutboxReplayResult = { + available: boolean; + cursor: string; + scanned: number; + replayed: number; + discarded: number; + retained: number; + backlog: number | null; +}; + +type RoutingTraceOutboxSchedulerState = { + cursor: string; + intervalId: ReturnType | null; + inFlight: Promise | null; + stopping: boolean; +}; + +const schedulerGlobal = globalThis as typeof globalThis & { + __CCH_ROUTING_TRACE_OUTBOX_SCHEDULER__?: RoutingTraceOutboxSchedulerState; + __CCH_STOP_ROUTING_TRACE_OUTBOX__?: (options?: { + wait?: boolean; + maxWaitMs?: number; + }) => Promise; +}; + +function getReadyRedis(): Redis | null { + const redis = getRedisClient({ allowWhenRateLimitDisabled: true }); + return redis?.status === "ready" ? redis : null; +} + +async function getReadyRedisForStage(): Promise { + const redis = getRedisClient({ allowWhenRateLimitDisabled: true }); + if (!redis || redis.status === "end") return null; + if (redis.status === "ready") return redis; + + return new Promise((resolve) => { + let settled = false; + const finish = (value: Redis | null) => { + if (settled) return; + settled = true; + clearTimeout(timer); + redis.removeListener("ready", onReady); + redis.removeListener("end", onUnavailable); + resolve(value); + }; + const onReady = () => finish(redis); + const onUnavailable = () => finish(null); + const timer = setTimeout( + () => finish(redis.status === "ready" ? redis : null), + REDIS_READY_WAIT_MS + ); + redis.once("ready", onReady); + redis.once("end", onUnavailable); + }); +} + +async function runRedisOperation(operation: Promise): Promise { + let timeoutId: ReturnType | null = null; + try { + return await Promise.race([ + operation, + new Promise((_, reject) => { + timeoutId = setTimeout( + () => reject(new Error("routing trace outbox Redis operation timed out")), + REDIS_OPERATION_TIMEOUT_MS + ); + }), + ]); + } finally { + if (timeoutId) clearTimeout(timeoutId); + } +} + +function logBacklogPressure(backlog: number): void { + if (backlog < BACKLOG_WARN_THRESHOLD) return; + const now = Date.now(); + if (now - lastBacklogLogAt < BACKLOG_LOG_INTERVAL_MS) return; + lastBacklogLogAt = now; + const context = { + backlog, + warnThreshold: BACKLOG_WARN_THRESHOLD, + errorThreshold: BACKLOG_ERROR_THRESHOLD, + }; + if (backlog >= BACKLOG_ERROR_THRESHOLD) { + logger.error("[RoutingTraceOutbox] Backlog requires intervention", context); + } else { + logger.warn("[RoutingTraceOutbox] Backlog is growing", context); + } +} + +function parseOutboxEntry(payload: string): RoutingTraceOutboxEntry | null { + try { + const value = JSON.parse(payload) as Partial; + if ( + value.version !== 1 || + !Number.isSafeInteger(value.requestId) || + (value.requestId ?? 0) <= 0 || + !Number.isFinite(value.traceUpdatedAt) + ) { + return null; + } + const routingTrace = normalizeRoutingTrace(value.routingTrace); + if (!routingTrace || routingTrace.updatedAt !== value.traceUpdatedAt) return null; + return { + version: 1, + requestId: value.requestId as number, + traceUpdatedAt: value.traceUpdatedAt as number, + routingTrace, + }; + } catch { + return null; + } +} + +async function deleteIfUnchanged( + redis: Redis, + receipt: RoutingTraceOutboxReceipt +): Promise { + try { + const deleted = await runRedisOperation( + redis.eval( + DELETE_IF_UNCHANGED_LUA, + 1, + ROUTING_TRACE_OUTBOX_KEY, + receipt.field, + receipt.payload + ) + ); + return Number(deleted) > 0; + } catch (error) { + logger.warn("[RoutingTraceOutbox] Failed to acknowledge entry", { + requestId: Number(receipt.field), + error: error instanceof Error ? error.message : String(error), + }); + return false; + } +} + +export async function stageRoutingTraceOutbox( + requestId: number, + routingTrace: RoutingTraceV1 +): Promise { + const normalized = normalizeRoutingTrace(routingTrace); + if (!normalized || !Number.isSafeInteger(requestId) || requestId <= 0) { + return null; + } + const redis = await getReadyRedisForStage(); + if (!redis) return null; + + const field = String(requestId); + const payload = JSON.stringify({ + version: 1, + requestId, + traceUpdatedAt: normalized.updatedAt, + routingTrace: normalized, + } satisfies RoutingTraceOutboxEntry); + try { + const staged = await runRedisOperation( + redis.eval( + STAGE_IF_NOT_OLDER_LUA, + 1, + ROUTING_TRACE_OUTBOX_KEY, + field, + String(normalized.updatedAt), + payload + ) + ); + return Number(staged) > 0 ? { field, payload } : null; + } catch (error) { + logger.warn("[RoutingTraceOutbox] Failed to stage entry", { + requestId, + error: error instanceof Error ? error.message : String(error), + }); + return null; + } +} + +export async function acknowledgeRoutingTraceOutbox( + receipt: RoutingTraceOutboxReceipt +): Promise { + const redis = getReadyRedis(); + if (!redis) return false; + return deleteIfUnchanged(redis, receipt); +} + +export async function persistRoutingTraceMonotonically( + requestId: number, + routingTrace: RoutingTraceV1 +): Promise { + const normalized = normalizeRoutingTrace(routingTrace); + if (!normalized || !Number.isSafeInteger(requestId) || requestId <= 0) { + return false; + } + const assignments = buildMonotonicRoutingTraceAssignments(normalized, { + routingTrace: sql`${messageRequest.routingTrace}`, + updatedAt: sql`${messageRequest.updatedAt}`, + }); + const rows = await getMessageWriterDb() + .update(messageRequest) + .set({ + routingTrace: assignments.routingTrace, + updatedAt: assignments.updatedAt, + }) + .where(and(eq(messageRequest.id, requestId), isNull(messageRequest.deletedAt))) + .returning({ id: messageRequest.id }); + return rows.length > 0; +} + +export async function replayRoutingTraceOutbox( + options: { cursor?: string; limit?: number } = {} +): Promise { + const redis = getReadyRedis(); + const result: RoutingTraceOutboxReplayResult = { + available: redis !== null, + cursor: options.cursor ?? "0", + scanned: 0, + replayed: 0, + discarded: 0, + retained: 0, + backlog: null, + }; + if (!redis) return result; + + const limit = Math.max(1, Math.floor(options.limit ?? DEFAULT_REPLAY_LIMIT)); + let page: [string, string[]]; + try { + page = (await runRedisOperation( + redis.hscan(ROUTING_TRACE_OUTBOX_KEY, result.cursor, "COUNT", limit) + )) as [string, string[]]; + } catch (error) { + logger.warn("[RoutingTraceOutbox] Failed to scan entries", { + error: error instanceof Error ? error.message : String(error), + }); + return result; + } + + result.cursor = page[0]; + const fieldsAndPayloads = page[1]; + // Redis COUNT is a hint and a page may be larger than requested. Process the + // complete returned page before advancing its cursor so no tail is skipped. + for (let index = 0; index + 1 < fieldsAndPayloads.length; index += 2) { + const field = fieldsAndPayloads[index]; + const payload = fieldsAndPayloads[index + 1]; + if (field === undefined || payload === undefined) continue; + result.scanned++; + const receipt = { field, payload } satisfies RoutingTraceOutboxReceipt; + const entry = parseOutboxEntry(payload); + if (!entry || field !== String(entry.requestId)) { + if (await deleteIfUnchanged(redis, receipt)) result.discarded++; + else result.retained++; + continue; + } + + try { + const targetExists = await persistRoutingTraceMonotonically( + entry.requestId, + entry.routingTrace + ); + if (targetExists) result.replayed++; + else result.discarded++; + if (!(await deleteIfUnchanged(redis, receipt))) result.retained++; + } catch (error) { + result.retained++; + logger.warn("[RoutingTraceOutbox] Replay failed; entry retained", { + requestId: entry.requestId, + error: error instanceof Error ? error.message : String(error), + }); + } + } + + try { + result.backlog = await runRedisOperation(redis.hlen(ROUTING_TRACE_OUTBOX_KEY)); + logBacklogPressure(result.backlog); + } catch (error) { + logger.warn("[RoutingTraceOutbox] Failed to read backlog", { + error: error instanceof Error ? error.message : String(error), + }); + } + return result; +} + +async function runScheduledReplay(state: RoutingTraceOutboxSchedulerState): Promise { + if (state.stopping || state.inFlight) return; + const task = (async () => { + try { + const result = await replayRoutingTraceOutbox({ cursor: state.cursor }); + if (!result.available) return; + state.cursor = result.cursor; + if (result.scanned > 0 || result.retained > 0) { + logger.info("[RoutingTraceOutbox] Replay cycle completed", result); + } + } catch (error) { + logger.warn("[RoutingTraceOutbox] Replay cycle failed", { + error: error instanceof Error ? error.message : String(error), + }); + } + })(); + state.inFlight = task; + try { + await task; + } finally { + if (state.inFlight === task) state.inFlight = null; + } +} + +export async function startRoutingTraceOutboxReplayScheduler(): Promise { + const existing = schedulerGlobal.__CCH_ROUTING_TRACE_OUTBOX_SCHEDULER__; + if (existing && !existing.stopping) return; + + const state: RoutingTraceOutboxSchedulerState = { + cursor: "0", + intervalId: null, + inFlight: null, + stopping: false, + }; + schedulerGlobal.__CCH_ROUTING_TRACE_OUTBOX_SCHEDULER__ = state; + schedulerGlobal.__CCH_STOP_ROUTING_TRACE_OUTBOX__ = stopRoutingTraceOutboxReplayScheduler; + // Recovery starts after migrations, but readiness must not wait for a slow + // outbox row. The scheduler owns and joins this task during shutdown. + void runScheduledReplay(state); + + state.intervalId = setInterval(() => { + void runScheduledReplay(state); + }, REPLAY_INTERVAL_MS); + state.intervalId.unref?.(); +} + +export async function stopRoutingTraceOutboxReplayScheduler( + options: { wait?: boolean; maxWaitMs?: number } = {} +): Promise { + const state = schedulerGlobal.__CCH_ROUTING_TRACE_OUTBOX_SCHEDULER__; + if (!state) return; + state.stopping = true; + if (state.intervalId) { + clearInterval(state.intervalId); + state.intervalId = null; + } + if (options.wait !== false && state.inFlight) { + const maxWaitMs = options.maxWaitMs; + if (maxWaitMs !== undefined && Number.isFinite(maxWaitMs) && maxWaitMs > 0) { + let timer: ReturnType | null = null; + try { + await Promise.race([ + state.inFlight, + new Promise((resolve) => { + timer = setTimeout(resolve, maxWaitMs); + timer.unref?.(); + }), + ]); + } finally { + if (timer) clearTimeout(timer); + } + } else { + await state.inFlight; + } + } + if (options.wait === false) return; + if (schedulerGlobal.__CCH_ROUTING_TRACE_OUTBOX_SCHEDULER__ === state) { + schedulerGlobal.__CCH_ROUTING_TRACE_OUTBOX_SCHEDULER__ = undefined; + } + if (schedulerGlobal.__CCH_STOP_ROUTING_TRACE_OUTBOX__ === stopRoutingTraceOutboxReplayScheduler) { + schedulerGlobal.__CCH_STOP_ROUTING_TRACE_OUTBOX__ = undefined; + } +} diff --git a/src/repository/routing-trace-persistence.ts b/src/repository/routing-trace-persistence.ts new file mode 100644 index 000000000..2192da3cf --- /dev/null +++ b/src/repository/routing-trace-persistence.ts @@ -0,0 +1,41 @@ +import "server-only"; + +import { type SQL, sql } from "drizzle-orm"; +import { normalizeRoutingTrace, type RoutingTraceV1 } from "@/types/routing-trace"; + +export type RoutingTraceAssignments = { + routingTrace: SQL; + updatedAt: SQL; +}; + +/** + * Build an idempotent assignment that never replaces a newer routing trace. + * RoutingTrace.updatedAt is a logical revision clock (strictly increasing for + * every mutation), so replay order does not affect the final stored value. + */ +export function buildMonotonicRoutingTraceAssignments( + routingTrace: RoutingTraceV1, + columns: { routingTrace: SQL; updatedAt: SQL } +): RoutingTraceAssignments { + const normalized = normalizeRoutingTrace(routingTrace); + if (!normalized) { + throw new Error("routing trace is invalid"); + } + + const serialized = JSON.stringify(normalized); + const storedRevision = sql` + COALESCE( + CASE + WHEN jsonb_typeof(${columns.routingTrace}->'updatedAt') = 'number' + THEN (${columns.routingTrace}->>'updatedAt')::numeric + END, + '-Infinity'::numeric + ) + `; + const shouldReplace = sql`${storedRevision} < ${normalized.updatedAt}::numeric`; + + return { + routingTrace: sql`CASE WHEN ${shouldReplace} THEN ${serialized}::jsonb ELSE ${columns.routingTrace} END`, + updatedAt: sql`CASE WHEN ${shouldReplace} THEN NOW() ELSE ${columns.updatedAt} END`, + }; +} diff --git a/tests/unit/lib/shutdown.test.ts b/tests/unit/lib/shutdown.test.ts index 264a1b682..70b80e612 100644 --- a/tests/unit/lib/shutdown.test.ts +++ b/tests/unit/lib/shutdown.test.ts @@ -21,6 +21,8 @@ describe.sequential("lifecycle/shutdown", () => { .__CCH_API_KEY_VF_SYNC_CLEANUP__; delete (globalThis as unknown as { __CCH_STOP_BACKGROUND_QUEUES__?: unknown }) .__CCH_STOP_BACKGROUND_QUEUES__; + delete (globalThis as unknown as { __CCH_STOP_ROUTING_TRACE_OUTBOX__?: unknown }) + .__CCH_STOP_ROUTING_TRACE_OUTBOX__; }); afterEach(() => { @@ -29,6 +31,8 @@ describe.sequential("lifecycle/shutdown", () => { delete (globalThis as unknown as { __ASYNC_TASK_MANAGER__?: unknown }).__ASYNC_TASK_MANAGER__; delete (globalThis as unknown as { __CCH_STOP_BACKGROUND_QUEUES__?: unknown }) .__CCH_STOP_BACKGROUND_QUEUES__; + delete (globalThis as unknown as { __CCH_STOP_ROUTING_TRACE_OUTBOX__?: unknown }) + .__CCH_STOP_ROUTING_TRACE_OUTBOX__; }); it("markShuttingDown flips isShuttingDown idempotently", async () => { @@ -262,6 +266,56 @@ describe.sequential("lifecycle/shutdown", () => { expect(writerStarted).toHaveBeenCalledTimes(1); }); + it("stops outbox ticks before writer shutdown but defers replay join until after writer", async () => { + let releaseReplay!: () => void; + const replaySettled = new Promise((resolve) => { + releaseReplay = resolve; + }); + const stopOutbox = vi.fn(async (options?: { wait?: boolean }) => { + if (options?.wait !== false) await replaySettled; + }); + const stopWriter = vi.fn(async () => {}); + const closeDbPools = vi.fn(async () => {}); + + vi.doMock("@/lib/cache/session-cache", () => ({ stopCacheCleanup: () => {} })); + ( + globalThis as unknown as { + __CCH_STOP_ROUTING_TRACE_OUTBOX__?: typeof stopOutbox; + } + ).__CCH_STOP_ROUTING_TRACE_OUTBOX__ = stopOutbox; + vi.doMock("@/lib/provider-endpoints/probe-scheduler", () => ({ + stopEndpointProbeScheduler: async () => {}, + })); + vi.doMock("@/lib/public-status/scheduler", () => ({ + stopPublicStatusRebuildScheduler: async () => {}, + })); + vi.doMock("@/lib/provider-endpoints/probe-log-cleanup", () => ({ + stopEndpointProbeLogCleanup: async () => {}, + })); + vi.doMock("@/lib/async-task-manager", () => ({ shutdownAllAsyncTasks: async () => {} })); + vi.doMock("@/repository/message-write-buffer", () => ({ + stopMessageRequestWriteBuffer: stopWriter, + })); + vi.doMock("@/drizzle/db", () => ({ closeDbPools })); + vi.doMock("@/lib/langfuse", () => ({ shutdownLangfuse: async () => {} })); + vi.doMock("@/lib/redis", () => ({ closeRedis: async () => {} })); + + const { runApplicationCleanup } = await import("@/lib/lifecycle/shutdown"); + const cleanup = runApplicationCleanup("SIGTERM", { + totalTimeoutMs: 5_000, + perStepTimeoutMs: 500, + }); + + await vi.waitFor(() => expect(stopWriter).toHaveBeenCalledOnce()); + expect(stopOutbox).toHaveBeenNthCalledWith(1, { wait: false }); + expect(stopOutbox).toHaveBeenNthCalledWith(2, { wait: true, maxWaitMs: 500 }); + expect(closeDbPools).not.toHaveBeenCalled(); + + releaseReplay(); + await cleanup; + expect(closeDbPools).toHaveBeenCalledOnce(); + }); + it("continues critical cleanup after background queue shutdown fails", async () => { const queueError = new Error("queue stop failed"); const shutdownTasks = vi.fn(async () => {}); diff --git a/tests/unit/proxy/routing-trace.test.ts b/tests/unit/proxy/routing-trace.test.ts index a6c1dcacc..4950e3d0f 100644 --- a/tests/unit/proxy/routing-trace.test.ts +++ b/tests/unit/proxy/routing-trace.test.ts @@ -161,6 +161,35 @@ describe("ProxySession routing trace recorder", () => { }); }); + it("advances the persistence revision when terminal events share one wall-clock millisecond", () => { + const nowSpy = vi.spyOn(Date, "now").mockReturnValue(1_000); + const session = makeTraceSession(); + session.initializeRoutingTrace({ + mode: "discovery", + discoveryEnabled: true, + eligible: true, + startedAt: 1_000, + }); + const initializedRevision = session.getRoutingTrace()?.updatedAt ?? 0; + + session.finalizeRoutingTrace(200, "success"); + const terminalRevision = session.getRoutingTrace()?.updatedAt ?? 0; + session.appendRoutingTraceEvent({ + type: "binding_finalized", + bindingAction: "create", + outcome: "updated", + }); + const bindingRevision = session.getRoutingTrace()?.updatedAt ?? 0; + + expect(terminalRevision).toBeGreaterThan(initializedRevision); + expect(bindingRevision).toBeGreaterThan(terminalRevision); + expect(session.getRoutingTrace()?.events.at(-1)).toMatchObject({ + type: "binding_finalized", + at: 1_000, + }); + nowSpy.mockRestore(); + }); + it("caps the trace at 512 events and persists the truncated snapshot independently", async () => { const session = makeTraceSession(); session.initializeRoutingTrace({ diff --git a/tests/unit/repository/message-terminal-write-apis.test.ts b/tests/unit/repository/message-terminal-write-apis.test.ts index 6c14bb8d4..970b61226 100644 --- a/tests/unit/repository/message-terminal-write-apis.test.ts +++ b/tests/unit/repository/message-terminal-write-apis.test.ts @@ -12,6 +12,13 @@ function installSyncBoundaries(insertedRows: readonly Record[] const updateSet = vi.fn((_values: Record) => ({ where: updateWhere })); const update = vi.fn((_table: unknown) => ({ set: updateSet })); const writerUpdate = vi.fn((_table: unknown) => ({ set: updateSet })); + const outboxReceipt = { field: "706", payload: "routing-trace-706" }; + const stageRoutingTraceOutbox = vi.fn(async () => outboxReceipt); + const acknowledgeRoutingTraceOutbox = vi.fn(async () => true); + const persistRoutingTraceMonotonically = vi.fn(async (_id: number, routingTrace: unknown) => { + await updateSet({ routingTrace, updatedAt: new Date() }).where(updateWhere); + return true; + }); vi.doMock("@/drizzle/db", () => ({ db: { insert, update, select: vi.fn(), execute: vi.fn() }, @@ -21,8 +28,21 @@ function installSyncBoundaries(insertedRows: readonly Record[] getEnvConfig: vi.fn(() => ({ MESSAGE_REQUEST_WRITE_MODE: "sync" as const })), isDevelopment: vi.fn(() => false), })); + vi.doMock("@/repository/routing-trace-outbox", () => ({ + acknowledgeRoutingTraceOutbox, + persistRoutingTraceMonotonically, + stageRoutingTraceOutbox, + })); - return { insertValues, update, updateSet, updateWhere }; + return { + acknowledgeRoutingTraceOutbox, + insertValues, + persistRoutingTraceMonotonically, + stageRoutingTraceOutbox, + update, + updateSet, + updateWhere, + }; } function installAsyncRoutingTraceBoundaries() { @@ -31,6 +51,10 @@ function installAsyncRoutingTraceBoundaries() { const enqueueMessageRequestUpdate = vi.fn(); const enqueueMessageRequestPostTerminalRoutingTraceDurably = vi.fn(async () => true); const loggerWarn = vi.fn(); + const outboxReceipt = { field: "707", payload: "routing-trace-707" }; + const stageRoutingTraceOutbox = vi.fn(async () => outboxReceipt); + const acknowledgeRoutingTraceOutbox = vi.fn(async () => true); + const persistRoutingTraceMonotonically = vi.fn(async () => true); vi.doMock("@/drizzle/db", () => ({ db: { @@ -60,13 +84,22 @@ function installAsyncRoutingTraceBoundaries() { warn: loggerWarn, }, })); + vi.doMock("@/repository/routing-trace-outbox", () => ({ + acknowledgeRoutingTraceOutbox, + persistRoutingTraceMonotonically, + stageRoutingTraceOutbox, + })); return { + acknowledgeRoutingTraceOutbox, controlUpdate, enqueueMessageRequestPostTerminalRoutingTraceDurably, enqueueMessageRequestUpdate, getMessageWriterDb, loggerWarn, + outboxReceipt, + persistRoutingTraceMonotonically, + stageRoutingTraceOutbox, }; } @@ -88,6 +121,7 @@ describe("message terminal write APIs", () => { vi.doUnmock("@/lib/config/env.schema"); vi.doUnmock("@/lib/logger"); vi.doUnmock("@/repository/message-write-buffer"); + vi.doUnmock("@/repository/routing-trace-outbox"); }); it("creates a request through the repository and returns its public row", async () => { @@ -164,14 +198,17 @@ describe("message terminal write APIs", () => { it("writes duration through the synchronous database boundary", async () => { vi.resetModules(); - const { updateSet, updateWhere } = installSyncBoundaries(); + const boundaries = installSyncBoundaries(); const { updateMessageRequestDuration } = await import("@/repository/message"); const result = await updateMessageRequestDuration(702, 345); expect(result).toBeUndefined(); - expect(updateSet).toHaveBeenCalledWith({ durationMs: 345, updatedAt: expect.any(Date) }); - expect(updateWhere).toHaveBeenCalledTimes(1); + expect(boundaries.updateSet).toHaveBeenCalledWith({ + durationMs: 345, + updatedAt: expect.any(Date), + }); + expect(boundaries.updateWhere).toHaveBeenCalledTimes(1); }); it("formats and writes the request cost", async () => { @@ -230,7 +267,7 @@ describe("message terminal write APIs", () => { it("patches a finalized routing trace without touching terminal or billing fields", async () => { vi.resetModules(); - const { updateSet, updateWhere } = installSyncBoundaries(); + const boundaries = installSyncBoundaries(); const { updateMessageRequestRoutingTrace } = await import("@/repository/message"); const routingTrace = { version: 1 as const, @@ -245,13 +282,16 @@ describe("message terminal write APIs", () => { await updateMessageRequestRoutingTrace(706, routingTrace); - expect(updateSet).toHaveBeenCalledWith({ + expect(boundaries.updateSet).toHaveBeenCalledWith({ routingTrace, updatedAt: expect.any(Date), }); - expect(updateSet.mock.calls[0]?.[0]).not.toHaveProperty("statusCode"); - expect(updateSet.mock.calls[0]?.[0]).not.toHaveProperty("costUsd"); - expect(updateWhere).toHaveBeenCalledTimes(1); + expect(boundaries.updateSet.mock.calls[0]?.[0]).not.toHaveProperty("statusCode"); + expect(boundaries.updateSet.mock.calls[0]?.[0]).not.toHaveProperty("costUsd"); + expect(boundaries.updateWhere).toHaveBeenCalledTimes(1); + expect(boundaries.stageRoutingTraceOutbox).toHaveBeenCalledWith(706, routingTrace); + expect(boundaries.persistRoutingTraceMonotonically).toHaveBeenCalledWith(706, routingTrace); + expect(boundaries.acknowledgeRoutingTraceOutbox).toHaveBeenCalledOnce(); }); it("patches an async finalized routing trace through acknowledged post-terminal metadata", async () => { @@ -283,8 +323,14 @@ describe("message terminal write APIs", () => { expect(boundaries.getMessageWriterDb).not.toHaveBeenCalled(); expect(boundaries.enqueueMessageRequestPostTerminalRoutingTraceDurably).toHaveBeenCalledWith( 707, - routingTrace + routingTrace, + { timeoutMs: 3_000 } + ); + expect(boundaries.stageRoutingTraceOutbox).toHaveBeenCalledWith(707, routingTrace); + expect(boundaries.stageRoutingTraceOutbox.mock.invocationCallOrder[0]).toBeLessThan( + boundaries.enqueueMessageRequestPostTerminalRoutingTraceDurably.mock.invocationCallOrder[0]! ); + expect(boundaries.acknowledgeRoutingTraceOutbox).toHaveBeenCalledWith(boundaries.outboxReceipt); expect(boundaries.loggerWarn).not.toHaveBeenCalled(); }); @@ -313,9 +359,104 @@ describe("message terminal write APIs", () => { "[MessageRequest] Failed to patch finalized routing trace", { requestId: 709, + recoverable: true, error: "durable writer unavailable", } ); + expect(boundaries.acknowledgeRoutingTraceOutbox).not.toHaveBeenCalled(); + }); + + it("reports an unrecoverable boundary when both outbox staging and the writer fail", async () => { + vi.resetModules(); + const boundaries = installAsyncRoutingTraceBoundaries(); + boundaries.stageRoutingTraceOutbox.mockResolvedValue(null); + boundaries.enqueueMessageRequestPostTerminalRoutingTraceDurably.mockRejectedValue( + new Error("writer unavailable") + ); + boundaries.persistRoutingTraceMonotonically.mockRejectedValue( + new Error("direct writer unavailable") + ); + const { updateMessageRequestRoutingTrace } = await import("@/repository/message"); + + await updateMessageRequestRoutingTrace(712, { + version: 1, + mode: "discovery", + startedAt: 1_000, + updatedAt: 1_100, + discoveryEnabled: true, + eligible: true, + events: [], + }); + + expect(boundaries.loggerWarn).toHaveBeenCalledWith( + "[MessageRequest] Failed to patch finalized routing trace", + { + requestId: 712, + recoverable: false, + error: "direct writer unavailable", + } + ); + expect(boundaries.acknowledgeRoutingTraceOutbox).not.toHaveBeenCalled(); + }); + + it("uses a monotonic direct fallback when coalescing occurs without an outbox receipt", async () => { + vi.resetModules(); + const boundaries = installAsyncRoutingTraceBoundaries(); + boundaries.stageRoutingTraceOutbox.mockResolvedValue(null); + boundaries.enqueueMessageRequestPostTerminalRoutingTraceDurably.mockResolvedValue(false); + const { updateMessageRequestRoutingTrace } = await import("@/repository/message"); + const routingTrace = { + version: 1 as const, + mode: "discovery" as const, + startedAt: 1_000, + updatedAt: 1_102, + discoveryEnabled: true, + eligible: true, + events: [], + }; + + await updateMessageRequestRoutingTrace(713, routingTrace); + + expect(boundaries.persistRoutingTraceMonotonically).toHaveBeenCalledWith(713, routingTrace); + expect(boundaries.loggerWarn).not.toHaveBeenCalled(); + expect(boundaries.acknowledgeRoutingTraceOutbox).not.toHaveBeenCalled(); + }); + + it("keeps a newer outbox receipt when the writer only committed an older coalesced trace", async () => { + vi.resetModules(); + const boundaries = installAsyncRoutingTraceBoundaries(); + const oldReceipt = { field: "711", payload: "old" }; + const newReceipt = { field: "711", payload: "new" }; + boundaries.stageRoutingTraceOutbox + .mockResolvedValueOnce(oldReceipt) + .mockResolvedValueOnce(newReceipt); + boundaries.enqueueMessageRequestPostTerminalRoutingTraceDurably + .mockResolvedValueOnce(true) + .mockResolvedValueOnce(false); + const { updateMessageRequestRoutingTrace } = await import("@/repository/message"); + + await updateMessageRequestRoutingTrace(711, { + version: 1, + mode: "discovery", + startedAt: 1_000, + updatedAt: 1_100, + discoveryEnabled: true, + eligible: true, + events: [], + }); + await updateMessageRequestRoutingTrace(711, { + version: 1, + mode: "discovery", + startedAt: 1_000, + updatedAt: 1_101, + discoveryEnabled: true, + eligible: true, + events: [], + }); + + expect(boundaries.acknowledgeRoutingTraceOutbox).toHaveBeenCalledOnce(); + expect(boundaries.acknowledgeRoutingTraceOutbox).toHaveBeenCalledWith(oldReceipt); + expect(boundaries.acknowledgeRoutingTraceOutbox).not.toHaveBeenCalledWith(newReceipt); }); it("logs and skips an invalid routing trace without persisting raw trace data", async () => { @@ -328,6 +469,7 @@ describe("message terminal write APIs", () => { >[1]); expect(boundaries.enqueueMessageRequestPostTerminalRoutingTraceDurably).not.toHaveBeenCalled(); + expect(boundaries.stageRoutingTraceOutbox).not.toHaveBeenCalled(); expect(boundaries.controlUpdate).not.toHaveBeenCalled(); expect(boundaries.loggerWarn).toHaveBeenCalledWith( "[MessageRequest] Skipped patching invalid routing trace", diff --git a/tests/unit/repository/message-write-buffer.test.ts b/tests/unit/repository/message-write-buffer.test.ts index 09832d09d..a17b5e78a 100644 --- a/tests/unit/repository/message-write-buffer.test.ts +++ b/tests/unit/repository/message-write-buffer.test.ts @@ -191,6 +191,27 @@ describe("message_request 异步批量写入", () => { expect(defaultExecuteMock).not.toHaveBeenCalled(); }); + it("非法 routing trace 应写入 SQL NULL 而不是 JSON null", async () => { + process.env.MESSAGE_REQUEST_WRITE_MODE = "async"; + + const { + enqueueMessageRequestUpdate, + flushMessageRequestWriteBuffer, + stopMessageRequestWriteBuffer, + } = await import("@/repository/message-write-buffer"); + + enqueueMessageRequestUpdate(44, { + routingTrace: { version: 999, events: [] } as never, + }); + await flushMessageRequestWriteBuffer(); + await stopMessageRequestWriteBuffer(); + + const built = toSqlText(executeMock.mock.calls[0]?.[0]); + expect(built.sql).toContain('"routing_trace" = CASE id'); + expect(built.sql).toContain("THEN NULL"); + expect(built.params).not.toContain("null"); + }); + it("普通 enqueue 立即返回，但 durable enqueue 应等待批量 SQL 成功", async () => { process.env.MESSAGE_REQUEST_WRITE_MODE = "async"; @@ -278,6 +299,11 @@ describe("message_request 异步批量写入", () => { const built = toSqlText(executeMock.mock.calls[0]?.[0]); expect(built.sql).toContain("RETURNING id"); expect(built.sql).toContain("routing_trace"); + expect(built.sql).toContain("jsonb_typeof"); + expect(built.sql).toContain("'updatedAt'"); + expect(built.sql).toContain("'-Infinity'::numeric"); + expect(built.sql).toContain('"updated_at" = CASE id'); + expect(built.sql).toContain('ELSE "updated_at" END'); expect(built.sql).not.toContain("status_code IS NULL"); expect(built.sql).not.toContain("duration_ms"); expect( @@ -462,6 +488,37 @@ describe("message_request 异步批量写入", () => { await stopMessageRequestWriteBuffer(); }); + it("does not acknowledge a newer revision through an older in-flight metadata task", async () => { + process.env.MESSAGE_REQUEST_WRITE_MODE = "async"; + const releaseOld = createDeferred(); + executeMock.mockImplementationOnce(async () => releaseOld.promise); + + const { + enqueueMessageRequestPostTerminalRoutingTraceDurably, + flushMessageRequestWriteBuffer, + stopMessageRequestWriteBuffer, + } = await import("@/repository/message-write-buffer"); + const oldTrace = createRoutingTrace(7_100); + const newTrace = createRoutingTrace(7_101); + const oldWrite = enqueueMessageRequestPostTerminalRoutingTraceDurably(52_107, oldTrace); + const flush = flushMessageRequestWriteBuffer(); + const newerWrite = enqueueMessageRequestPostTerminalRoutingTraceDurably(52_107, newTrace); + + releaseOld.resolve([{ id: 52_107 }]); + await flush; + await expect(oldWrite).resolves.toBe(true); + await expect(newerWrite).resolves.toBe(false); + expect(executeMock).toHaveBeenCalledOnce(); + const built = toSqlText(executeMock.mock.calls[0]?.[0]); + expect( + built.params.some((value) => typeof value === "string" && value.includes('"updatedAt":7100')) + ).toBe(true); + expect( + built.params.some((value) => typeof value === "string" && value.includes('"updatedAt":7101')) + ).toBe(false); + await stopMessageRequestWriteBuffer(); + }); + it("flushes an existing ordinary patch before isolating post-terminal metadata", async () => { process.env.MESSAGE_REQUEST_WRITE_MODE = "async"; diff --git a/tests/unit/repository/routing-trace-outbox.test.ts b/tests/unit/repository/routing-trace-outbox.test.ts new file mode 100644 index 000000000..4a3942ed1 --- /dev/null +++ b/tests/unit/repository/routing-trace-outbox.test.ts @@ -0,0 +1,318 @@ +import type { SQL } from "drizzle-orm"; +import { PgDialect } from "drizzle-orm/pg-core"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { RoutingTraceV1 } from "@/types/routing-trace"; + +const mocks = vi.hoisted(() => { + const hash = new Map(); + const redis = { + status: "ready", + eval: vi.fn(), + hlen: vi.fn(), + hscan: vi.fn(), + once: vi.fn(), + removeListener: vi.fn(), + }; + const returning = vi.fn(); + const where = vi.fn(() => ({ returning })); + const set = vi.fn(() => ({ where })); + const update = vi.fn(() => ({ set })); + const getMessageWriterDb = vi.fn(() => ({ update })); + return { + getMessageWriterDb, + hash, + redis, + returning, + set, + update, + where, + }; +}); + +vi.mock("@/lib/redis/client", () => ({ + getRedisClient: vi.fn(() => mocks.redis), +})); + +vi.mock("@/drizzle/db", () => ({ + getMessageWriterDb: mocks.getMessageWriterDb, +})); + +vi.mock("@/lib/logger", () => ({ + logger: { + debug: vi.fn(), + error: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + }, +})); + +import { + acknowledgeRoutingTraceOutbox, + replayRoutingTraceOutbox, + stageRoutingTraceOutbox, + startRoutingTraceOutboxReplayScheduler, + stopRoutingTraceOutboxReplayScheduler, +} from "@/repository/routing-trace-outbox"; + +function createTrace(updatedAt: number): RoutingTraceV1 { + return { + version: 1, + mode: "discovery", + startedAt: 1_000, + updatedAt, + discoveryEnabled: true, + eligible: true, + events: [ + { + type: "binding_finalized", + at: updatedAt, + elapsedMs: updatedAt - 1_000, + bindingAction: "create", + outcome: "updated", + }, + ], + }; +} + +function toSqlText(value: SQL): { sql: string; params: unknown[] } { + return new PgDialect().sqlToQuery(value); +} + +describe("routing trace outbox", () => { + beforeEach(async () => { + await stopRoutingTraceOutboxReplayScheduler(); + vi.useRealTimers(); + vi.clearAllMocks(); + mocks.hash.clear(); + mocks.redis.status = "ready"; + mocks.returning.mockResolvedValue([{ id: 41 }]); + mocks.redis.eval.mockImplementation( + async (script: string, _keyCount: number, _key: string, ...args: string[]) => { + if (script.includes("HSET")) { + const [field, incomingRevisionRaw, payload] = args; + if (!field || !incomingRevisionRaw || !payload) return 0; + const current = mocks.hash.get(field); + if (current) { + const currentRevision = Number( + (JSON.parse(current) as { traceUpdatedAt?: unknown }).traceUpdatedAt + ); + const incomingRevision = Number(incomingRevisionRaw); + if (currentRevision > incomingRevision) return 0; + if (currentRevision === incomingRevision && current !== payload) return 0; + } + mocks.hash.set(field, payload); + return 1; + } + if (script.includes("HDEL")) { + const [field, payload] = args; + if (!field || !payload || mocks.hash.get(field) !== payload) return 0; + mocks.hash.delete(field); + return 1; + } + throw new Error("unexpected Lua script"); + } + ); + mocks.redis.hscan.mockImplementation( + async (_key: string, cursor: string, _countKeyword: string, count: number) => { + const entries = Array.from(mocks.hash.entries()); + const start = Number(cursor); + const page = entries.slice(start, start + count); + const nextCursor = + start + page.length >= entries.length ? "0" : String(start + page.length); + return [nextCursor, page.flat()]; + } + ); + mocks.redis.hlen.mockImplementation(async () => mocks.hash.size); + }); + + afterEach(async () => { + await stopRoutingTraceOutboxReplayScheduler(); + vi.useRealTimers(); + }); + + it("stages the normalized trace before persistence without an expiry", async () => { + const receipt = await stageRoutingTraceOutbox(41, createTrace(1_100)); + + expect(receipt).toEqual({ field: "41", payload: mocks.hash.get("41") }); + expect(JSON.parse(receipt?.payload ?? "{}")).toMatchObject({ + version: 1, + requestId: 41, + traceUpdatedAt: 1_100, + }); + expect(String(mocks.redis.eval.mock.calls[0]?.[0])).not.toContain("EXPIRE"); + }); + + it("does not let an older stage replace a newer recoverable snapshot", async () => { + const newReceipt = await stageRoutingTraceOutbox(41, createTrace(1_101)); + + await expect(stageRoutingTraceOutbox(41, createTrace(1_100))).resolves.toBeNull(); + expect(mocks.hash.get("41")).toBe(newReceipt?.payload); + }); + + it("waits briefly for an initial Redis connection before staging", async () => { + vi.useFakeTimers(); + mocks.redis.status = "connecting"; + const staged = stageRoutingTraceOutbox(41, createTrace(1_100)); + + mocks.redis.status = "ready"; + await vi.advanceTimersByTimeAsync(500); + + await expect(staged).resolves.toEqual({ field: "41", payload: mocks.hash.get("41") }); + }); + + it("bounds a stalled Redis stage operation", async () => { + vi.useFakeTimers(); + mocks.redis.eval.mockImplementationOnce(async () => new Promise(() => {})); + const staged = stageRoutingTraceOutbox(41, createTrace(1_100)); + + await vi.advanceTimersByTimeAsync(1_000); + + await expect(staged).resolves.toBeNull(); + }); + + it("acknowledges only the exact staged payload", async () => { + const oldReceipt = await stageRoutingTraceOutbox(41, createTrace(1_100)); + const newReceipt = await stageRoutingTraceOutbox(41, createTrace(1_101)); + + await expect(acknowledgeRoutingTraceOutbox(oldReceipt!)).resolves.toBe(false); + expect(mocks.hash.get("41")).toBe(newReceipt?.payload); + await expect(acknowledgeRoutingTraceOutbox(newReceipt!)).resolves.toBe(true); + expect(mocks.hash.has("41")).toBe(false); + }); + + it("replays a trace monotonically and removes it after database acknowledgement", async () => { + await stageRoutingTraceOutbox(41, createTrace(1_100)); + + const result = await replayRoutingTraceOutbox(); + + expect(result).toMatchObject({ replayed: 1, retained: 0, backlog: 0 }); + expect(mocks.hash.has("41")).toBe(false); + const assignment = mocks.set.mock.calls[0]?.[0] as { + routingTrace: SQL; + updatedAt: SQL; + }; + const built = toSqlText(assignment.routingTrace); + expect(built.sql).toContain("jsonb_typeof"); + expect(built.sql).toContain("<"); + expect(built.params).toContain(1_100); + }); + + it("retains the staged trace when the database write fails", async () => { + mocks.returning.mockRejectedValueOnce(new Error("database unavailable")); + const receipt = await stageRoutingTraceOutbox(41, createTrace(1_100)); + + const result = await replayRoutingTraceOutbox(); + + expect(result).toMatchObject({ replayed: 0, retained: 1, backlog: 1 }); + expect(mocks.hash.get("41")).toBe(receipt?.payload); + }); + + it("keeps a newer stage when an older replay acknowledgement arrives late", async () => { + let releaseOldWrite!: (rows: Array<{ id: number }>) => void; + const oldWrite = new Promise>((resolve) => { + releaseOldWrite = resolve; + }); + mocks.returning.mockImplementationOnce(async () => oldWrite); + await stageRoutingTraceOutbox(41, createTrace(1_100)); + + const replay = replayRoutingTraceOutbox(); + await vi.waitFor(() => expect(mocks.returning).toHaveBeenCalledOnce()); + const newReceipt = await stageRoutingTraceOutbox(41, createTrace(1_101)); + releaseOldWrite([{ id: 41 }]); + const oldResult = await replay; + + expect(oldResult.retained).toBe(1); + expect(mocks.hash.get("41")).toBe(newReceipt?.payload); + await replayRoutingTraceOutbox(); + expect(mocks.hash.has("41")).toBe(false); + }); + + it("continues a bounded HSCAN cursor across replay cycles", async () => { + await Promise.all([ + stageRoutingTraceOutbox(41, createTrace(1_100)), + stageRoutingTraceOutbox(42, createTrace(1_100)), + stageRoutingTraceOutbox(43, createTrace(1_100)), + ]); + mocks.returning.mockRejectedValue(new Error("database unavailable")); + + const first = await replayRoutingTraceOutbox({ limit: 2 }); + const second = await replayRoutingTraceOutbox({ cursor: first.cursor, limit: 2 }); + + expect(first).toMatchObject({ scanned: 2, retained: 2, backlog: 3 }); + expect(first.cursor).not.toBe("0"); + expect(second).toMatchObject({ scanned: 1, retained: 1, backlog: 3 }); + expect(second.cursor).toBe("0"); + }); + + it("processes an entire HSCAN page when Redis returns more than the COUNT hint", async () => { + await Promise.all([ + stageRoutingTraceOutbox(41, createTrace(1_100)), + stageRoutingTraceOutbox(42, createTrace(1_100)), + stageRoutingTraceOutbox(43, createTrace(1_100)), + ]); + mocks.redis.hscan.mockResolvedValueOnce(["0", Array.from(mocks.hash.entries()).flat()]); + + const result = await replayRoutingTraceOutbox({ limit: 2 }); + + expect(result).toMatchObject({ scanned: 3, replayed: 3, backlog: 0 }); + }); + + it("eventually drains a backlog larger than one replay page", async () => { + await Promise.all( + Array.from({ length: 250 }, (_, index) => + stageRoutingTraceOutbox(1_000 + index, createTrace(1_100 + index)) + ) + ); + + let cursor = "0"; + for (let cycle = 0; cycle < 5 && mocks.hash.size > 0; cycle++) { + const result = await replayRoutingTraceOutbox({ cursor, limit: 100 }); + cursor = result.cursor; + } + + expect(mocks.hash.size).toBe(0); + expect(mocks.returning).toHaveBeenCalledTimes(250); + }); + + it("discards malformed entries without sending them to the database", async () => { + mocks.hash.set("41", "not-json"); + + const result = await replayRoutingTraceOutbox(); + + expect(result).toMatchObject({ discarded: 1, retained: 0, backlog: 0 }); + expect(mocks.getMessageWriterDb).not.toHaveBeenCalled(); + }); + + it("retries after Redis becomes ready and stops future scheduler ticks", async () => { + vi.useFakeTimers(); + mocks.redis.status = "connecting"; + await startRoutingTraceOutboxReplayScheduler(); + expect(mocks.redis.hscan).not.toHaveBeenCalled(); + + mocks.redis.status = "ready"; + await vi.advanceTimersByTimeAsync(30_000); + expect(mocks.redis.hscan).toHaveBeenCalledOnce(); + + await stopRoutingTraceOutboxReplayScheduler(); + await vi.advanceTimersByTimeAsync(60_000); + expect(mocks.redis.hscan).toHaveBeenCalledOnce(); + }); + + it("stops ticks immediately and bounds the join of an in-flight replay", async () => { + vi.useFakeTimers(); + mocks.redis.hscan.mockImplementationOnce(async () => new Promise(() => {})); + await startRoutingTraceOutboxReplayScheduler(); + await Promise.resolve(); + expect(mocks.redis.hscan).toHaveBeenCalledOnce(); + + await expect(stopRoutingTraceOutboxReplayScheduler({ wait: false })).resolves.toBeUndefined(); + const joined = stopRoutingTraceOutboxReplayScheduler({ wait: true, maxWaitMs: 100 }); + await vi.advanceTimersByTimeAsync(100); + await expect(joined).resolves.toBeUndefined(); + + // Let the Redis operation's own timeout settle the detached cycle. + await vi.advanceTimersByTimeAsync(900); + await vi.advanceTimersByTimeAsync(60_000); + expect(mocks.redis.hscan).toHaveBeenCalledOnce(); + }); +}); From 5f149739e559a91ae59ee559d1e6b701434ac719 Mon Sep 17 00:00:00 2001 From: Brisbanehuang Date: Tue, 21 Jul 2026 11:20:26 -0400 Subject: [PATCH 07/12] feat(ui): expose Discovery attempt details --- messages/en/dashboard.json | 28 +- messages/en/settings/config.json | 2 + messages/ja/dashboard.json | 28 +- messages/ja/settings/config.json | 2 + messages/ru/dashboard.json | 28 +- messages/ru/settings/config.json | 2 + messages/zh-CN/dashboard.json | 28 +- messages/zh-CN/settings/config.json | 2 + messages/zh-TW/dashboard.json | 28 +- messages/zh-TW/settings/config.json | 2 + .../_components/error-details-dialog.test.tsx | 211 ++++++++++++++ .../components/DiscoveryTraceView.tsx | 266 +++++++++++++++--- .../components/LogicTraceTab.tsx | 14 +- .../provider-chain-popover.test.tsx | 84 ++++++ .../_components/provider-chain-popover.tsx | 50 +++- .../_components/system-settings-form.tsx | 16 +- src/app/[locale]/settings/config/page.tsx | 3 + src/app/v1/_lib/proxy/forwarder.ts | 7 +- src/types/routing-trace.ts | 5 + 19 files changed, 746 insertions(+), 60 deletions(-) diff --git a/messages/en/dashboard.json b/messages/en/dashboard.json index 8422afa78..68f36437c 100644 --- a/messages/en/dashboard.json +++ b/messages/en/dashboard.json @@ -385,7 +385,8 @@ "discovery": "Bounded Discovery", "legacy_hedge": "Legacy Hedge", "legacy_serial": "Legacy serial fallback", - "single_upstream": "Single upstream" + "single_upstream": "Single upstream", + "lease_conflict": "Single-route protection" }, "bypassed": "Discovery was not used: {reason}", "bypassReasons": { @@ -403,7 +404,7 @@ "rollout_ineligible": "Request is outside the current rollout", "redis_capability_unavailable": "Redis binding capability is unavailable", "binding_conflict": "Session binding conflict", - "lease_conflict": "Another request owns the Discovery lease", + "lease_conflict": "Another request owns the Discovery lease; this request uses one upstream", "lease_unavailable": "Discovery lease is unavailable", "unknown": "Eligibility requirement was not met" }, @@ -471,10 +472,31 @@ "configDiscoverySla": "Round SLA", "configStickySla": "Sticky SLA", "configTotalTimeout": "Total timeout", - "configStickyCooldown": "Sticky timeout cooldown" + "configStickyCooldown": "Sticky timeout cooldown", + "configStickyBindingTtl": "Sticky binding validity (SESSION_TTL)", + "attemptDetails": { + "providerId": "Provider ID", + "attempt": "Attempt", + "status": "HTTP status", + "inferred": "inferred", + "endpoint": "Endpoint", + "error": "Upstream error", + "cancellation": "Cancellation reason", + "timeline": "Attempt timeline" + }, + "cancellationKinds": { + "discovery_loser": "Another Discovery attempt won", + "discovery_sla_timeout": "Discovery SLA expired", + "round_timeout": "Round SLA expired", + "sticky_timeout": "Sticky SLA expired", + "request_deadline": "Total Discovery deadline reached", + "client_abort": "Client disconnected", + "winner_committed": "Another provider was committed" + } }, "logicTrace": { "title": "Decision Chain", + "singleRouteSelectionTitle": "Provider selection under single-route protection", "noDecisionData": "No decision data available", "providersCount": "{count} providers", "healthyCount": "{count} healthy", diff --git a/messages/en/settings/config.json b/messages/en/settings/config.json index 149408b5e..50a311026 100644 --- a/messages/en/settings/config.json +++ b/messages/en/settings/config.json @@ -126,6 +126,8 @@ "stickySlaMs": "Sticky SLA (milliseconds)", "racingTotalTimeoutMs": "Discovery total timeout (milliseconds)", "stickyTimeoutCooldownMs": "Sticky timeout cooldown (milliseconds)", + "stickyBindingTtl": "Sticky binding validity", + "stickyBindingTtlDesc": "Read-only. This follows the SESSION_TTL environment variable and is shared with other session snapshots.", "discoveryWindowDesc": "The total timeout must be at least Sticky SLA + maximum rounds × Discovery SLA. Discovery losers are cancelled and are not drained or billed by the legacy Hedge path.", "discoveryWindowInvalid": "Discovery total timeout is shorter than the configured Sticky and Discovery windows.", "discoverySettingsInvalid": "One or more Discovery values are outside the allowed range.", diff --git a/messages/ja/dashboard.json b/messages/ja/dashboard.json index 3b6be2ece..32e5feba1 100644 --- a/messages/ja/dashboard.json +++ b/messages/ja/dashboard.json @@ -385,7 +385,8 @@ "discovery": "限定 Discovery", "legacy_hedge": "従来の Hedge 競争", "legacy_serial": "従来の直列フェイルオーバー", - "single_upstream": "単一アップストリーム" + "single_upstream": "単一アップストリーム", + "lease_conflict": "単一経路保護" }, "bypassed": "このリクエストでは Discovery を使用していません：{reason}", "bypassReasons": { @@ -403,7 +404,7 @@ "rollout_ineligible": "現在のロールアウト対象外です", "redis_capability_unavailable": "Redis バインディング機能を利用できません", "binding_conflict": "Session バインディングが競合しています", - "lease_conflict": "別のリクエストが Discovery リースを保持しています", + "lease_conflict": "別のリクエストが Discovery リースを保持しているため、このリクエストは単一アップストリームを使用します", "lease_unavailable": "Discovery リースを利用できません", "unknown": "Discovery の適格条件を満たしていません" }, @@ -471,10 +472,31 @@ "configDiscoverySla": "ラウンド SLA", "configStickySla": "Sticky SLA", "configTotalTimeout": "全体タイムアウト", - "configStickyCooldown": "Sticky タイムアウトのクールダウン" + "configStickyCooldown": "Sticky タイムアウトのクールダウン", + "configStickyBindingTtl": "Sticky バインドの有効期間（SESSION_TTL）", + "attemptDetails": { + "providerId": "Provider ID", + "attempt": "試行回数", + "status": "HTTP ステータス", + "inferred": "推定", + "endpoint": "エンドポイント", + "error": "上流エラー", + "cancellation": "キャンセル理由", + "timeline": "試行タイムライン" + }, + "cancellationKinds": { + "discovery_loser": "別の Discovery 試行が勝利しました", + "discovery_sla_timeout": "Discovery SLA の期限に達しました", + "round_timeout": "ラウンド SLA の期限に達しました", + "sticky_timeout": "Sticky SLA の期限に達しました", + "request_deadline": "Discovery の合計期限に達しました", + "client_abort": "クライアントが切断しました", + "winner_committed": "別の Provider が勝者として確定しました" + } }, "logicTrace": { "title": "決定チェーン", + "singleRouteSelectionTitle": "単一経路保護での Provider 選択", "noDecisionData": "決定データがありません", "providersCount": "{count} プロバイダー", "healthyCount": "{count} 健全", diff --git a/messages/ja/settings/config.json b/messages/ja/settings/config.json index e49d28c1b..85a3b9e5b 100644 --- a/messages/ja/settings/config.json +++ b/messages/ja/settings/config.json @@ -128,6 +128,8 @@ "stickySlaMs": "Sticky SLA（ミリ秒）", "racingTotalTimeoutMs": "Discovery 合計タイムアウト（ミリ秒）", "stickyTimeoutCooldownMs": "Sticky タイムアウト後のクールダウン（ミリ秒）", + "stickyBindingTtl": "Sticky バインドの有効期間", + "stickyBindingTtlDesc": "読み取り専用です。この値は環境変数 SESSION_TTL に従い、他の Session スナップショットと共有されます。", "discoveryWindowDesc": "合計タイムアウトは Sticky SLA + 最大ラウンド数 × Discovery SLA 以上にしてください。", "discoveryWindowInvalid": "Discovery 合計タイムアウトが設定された Sticky/Discovery ウィンドウより短くなっています。", "discoverySettingsInvalid": "1 つ以上の Discovery 設定値が許容範囲外です。", diff --git a/messages/ru/dashboard.json b/messages/ru/dashboard.json index 1b1f66545..7088cc34b 100644 --- a/messages/ru/dashboard.json +++ b/messages/ru/dashboard.json @@ -385,7 +385,8 @@ "discovery": "Ограниченный Discovery", "legacy_hedge": "Прежняя гонка Hedge", "legacy_serial": "Прежнее последовательное переключение", - "single_upstream": "Один upstream" + "single_upstream": "Один upstream", + "lease_conflict": "Защита одним маршрутом" }, "bypassed": "Discovery не использован: {reason}", "bypassReasons": { @@ -403,7 +404,7 @@ "rollout_ineligible": "Запрос не входит в текущий rollout", "redis_capability_unavailable": "Возможность привязки Redis недоступна", "binding_conflict": "Конфликт привязки Session", - "lease_conflict": "Lease Discovery занят другим запросом", + "lease_conflict": "Lease Discovery занят другим запросом; этот запрос использует один upstream", "lease_unavailable": "Lease Discovery недоступен", "unknown": "Условия запуска Discovery не выполнены" }, @@ -471,10 +472,31 @@ "configDiscoverySla": "SLA раунда", "configStickySla": "SLA Sticky", "configTotalTimeout": "Общий тайм-аут", - "configStickyCooldown": "Пауза после тайм-аута Sticky" + "configStickyCooldown": "Пауза после тайм-аута Sticky", + "configStickyBindingTtl": "Срок действия привязки Sticky (SESSION_TTL)", + "attemptDetails": { + "providerId": "ID провайдера", + "attempt": "Попытка", + "status": "Статус HTTP", + "inferred": "выведен", + "endpoint": "Upstream", + "error": "Ошибка upstream", + "cancellation": "Причина отмены", + "timeline": "Хронология попытки" + }, + "cancellationKinds": { + "discovery_loser": "Победила другая попытка Discovery", + "discovery_sla_timeout": "SLA Discovery истёк", + "round_timeout": "SLA раунда истёк", + "sticky_timeout": "SLA Sticky истёк", + "request_deadline": "Общий срок Discovery истёк", + "client_abort": "Клиент отключился", + "winner_committed": "Другой провайдер был выбран победителем" + } }, "logicTrace": { "title": "Цепочка решений", + "singleRouteSelectionTitle": "Выбор провайдера в режиме защиты одним маршрутом", "noDecisionData": "Нет данных о решениях", "providersCount": "{count} поставщиков", "healthyCount": "{count} исправных", diff --git a/messages/ru/settings/config.json b/messages/ru/settings/config.json index 9624cf448..31aed5319 100644 --- a/messages/ru/settings/config.json +++ b/messages/ru/settings/config.json @@ -128,6 +128,8 @@ "stickySlaMs": "SLA Sticky (миллисекунды)", "racingTotalTimeoutMs": "Общий тайм-аут Discovery (миллисекунды)", "stickyTimeoutCooldownMs": "Пауза после тайм-аута Sticky (миллисекунды)", + "stickyBindingTtl": "Срок действия привязки Sticky", + "stickyBindingTtlDesc": "Только для чтения. Значение следует переменной окружения SESSION_TTL и используется другими снимками Session.", "discoveryWindowDesc": "Общий тайм-аут должен быть не меньше SLA Sticky + максимальное число раундов × SLA Discovery.", "discoveryWindowInvalid": "Общий тайм-аут Discovery меньше настроенного окна Sticky и Discovery.", "discoverySettingsInvalid": "Одно или несколько значений Discovery находятся вне допустимого диапазона.", diff --git a/messages/zh-CN/dashboard.json b/messages/zh-CN/dashboard.json index 94490ca05..0f61660b0 100644 --- a/messages/zh-CN/dashboard.json +++ b/messages/zh-CN/dashboard.json @@ -385,7 +385,8 @@ "discovery": "有界供应商 Discovery", "legacy_hedge": "旧版 Hedge 竞速", "legacy_serial": "旧版串行故障转移", - "single_upstream": "单上游模式" + "single_upstream": "单上游模式", + "lease_conflict": "单路保护" }, "bypassed": "本请求未使用 Discovery：{reason}", "bypassReasons": { @@ -403,7 +404,7 @@ "rollout_ineligible": "本请求不在当前灰度范围内", "redis_capability_unavailable": "Redis 绑定能力不可用", "binding_conflict": "Session 绑定状态冲突", - "lease_conflict": "另一个请求持有 Discovery 租约", + "lease_conflict": "另一个请求持有 Discovery 租约，本请求只使用一个上游", "lease_unavailable": "Discovery 租约不可用", "unknown": "未满足 Discovery 准入条件" }, @@ -471,10 +472,31 @@ "configDiscoverySla": "每轮 SLA", "configStickySla": "Sticky SLA", "configTotalTimeout": "总超时", - "configStickyCooldown": "Sticky 超时冷却" + "configStickyCooldown": "Sticky 超时冷却", + "configStickyBindingTtl": "Sticky 绑定有效期（SESSION_TTL）", + "attemptDetails": { + "providerId": "供应商 ID", + "attempt": "尝试次数", + "status": "HTTP 状态", + "inferred": "推断", + "endpoint": "端点", + "error": "上游错误", + "cancellation": "取消原因", + "timeline": "尝试时间线" + }, + "cancellationKinds": { + "discovery_loser": "其他 Discovery 尝试已胜出", + "discovery_sla_timeout": "Discovery SLA 已到期", + "round_timeout": "本轮 SLA 已到期", + "sticky_timeout": "Sticky SLA 已到期", + "request_deadline": "Discovery 总时限已到期", + "client_abort": "客户端已断开", + "winner_committed": "其他供应商已提交为赢家" + } }, "logicTrace": { "title": "决策链", + "singleRouteSelectionTitle": "单路保护下的供应商选择", "noDecisionData": "暂无决策数据", "providersCount": "{count} 个供应商", "healthyCount": "{count} 个健康", diff --git a/messages/zh-CN/settings/config.json b/messages/zh-CN/settings/config.json index c4a94ea93..acc8fc05d 100644 --- a/messages/zh-CN/settings/config.json +++ b/messages/zh-CN/settings/config.json @@ -55,6 +55,8 @@ "stickySlaMs": "Sticky SLA（毫秒）", "racingTotalTimeoutMs": "Discovery 总超时（毫秒）", "stickyTimeoutCooldownMs": "Sticky 超时冷却（毫秒）", + "stickyBindingTtl": "Sticky 绑定有效期", + "stickyBindingTtlDesc": "只读；该值跟随环境变量 SESSION_TTL，并与其他 Session 快照共用。", "discoveryWindowDesc": "总超时必须不小于 Sticky SLA + 最大轮数 × Discovery SLA。Discovery 输家会取消，不走旧 Hedge 的 drain 或输家计费。", "discoveryWindowInvalid": "Discovery 总超时短于已配置的 Sticky 与 Discovery 窗口。", "discoverySettingsInvalid": "一个或多个 Discovery 配置值超出允许范围。", diff --git a/messages/zh-TW/dashboard.json b/messages/zh-TW/dashboard.json index 34505e7fc..b2ebd14de 100644 --- a/messages/zh-TW/dashboard.json +++ b/messages/zh-TW/dashboard.json @@ -385,7 +385,8 @@ "discovery": "有界供應商 Discovery", "legacy_hedge": "舊版 Hedge 競速", "legacy_serial": "舊版串行故障轉移", - "single_upstream": "單上游模式" + "single_upstream": "單上游模式", + "lease_conflict": "單路保護" }, "bypassed": "本請求未使用 Discovery：{reason}", "bypassReasons": { @@ -403,7 +404,7 @@ "rollout_ineligible": "本請求不在目前灰度範圍內", "redis_capability_unavailable": "Redis 綁定能力不可用", "binding_conflict": "Session 綁定狀態衝突", - "lease_conflict": "另一個請求持有 Discovery 租約", + "lease_conflict": "另一個請求持有 Discovery 租約，本請求只使用一個上游", "lease_unavailable": "Discovery 租約不可用", "unknown": "未滿足 Discovery 准入條件" }, @@ -471,10 +472,31 @@ "configDiscoverySla": "每輪 SLA", "configStickySla": "Sticky SLA", "configTotalTimeout": "總逾時", - "configStickyCooldown": "Sticky 逾時冷卻" + "configStickyCooldown": "Sticky 逾時冷卻", + "configStickyBindingTtl": "Sticky 綁定有效期（SESSION_TTL）", + "attemptDetails": { + "providerId": "供應商 ID", + "attempt": "嘗試次數", + "status": "HTTP 狀態", + "inferred": "推斷", + "endpoint": "端點", + "error": "上游錯誤", + "cancellation": "取消緣由", + "timeline": "嘗試時間線" + }, + "cancellationKinds": { + "discovery_loser": "其他 Discovery 嘗試已勝出", + "discovery_sla_timeout": "Discovery SLA 已逾期", + "round_timeout": "本輪 SLA 已到期", + "sticky_timeout": "Sticky SLA 已逾期", + "request_deadline": "Discovery 總時限已到期", + "client_abort": "客戶端已斷開", + "winner_committed": "其他供應商已提交為贏家" + } }, "logicTrace": { "title": "決策鏈", + "singleRouteSelectionTitle": "單路保護下的供應商選擇", "noDecisionData": "暫無決策資料", "providersCount": "{count} 個供應商", "healthyCount": "{count} 個健康", diff --git a/messages/zh-TW/settings/config.json b/messages/zh-TW/settings/config.json index 613494c92..7a166f96c 100644 --- a/messages/zh-TW/settings/config.json +++ b/messages/zh-TW/settings/config.json @@ -128,6 +128,8 @@ "stickySlaMs": "Sticky 等待 SLA（毫秒）", "racingTotalTimeoutMs": "Discovery 總逾時（毫秒）", "stickyTimeoutCooldownMs": "Sticky 逾時冷卻（毫秒）", + "stickyBindingTtl": "Sticky 綁定有效期", + "stickyBindingTtlDesc": "唯讀；此值跟隨環境變數 SESSION_TTL，並與其他 Session 快照共用。", "discoveryWindowDesc": "總逾時必須不小於 Sticky SLA + 最大輪數 × Discovery SLA。Discovery 輸家會取消，不走舊 Hedge 的 drain 或輸家計費。", "discoveryWindowInvalid": "Discovery 總逾時短於已設定的 Sticky 與 Discovery 視窗。", "discoverySettingsInvalid": "一個或多個 Discovery 設定值超出允許範圍。", diff --git a/src/app/[locale]/dashboard/logs/_components/error-details-dialog.test.tsx b/src/app/[locale]/dashboard/logs/_components/error-details-dialog.test.tsx index 7267e7bd9..1d5f8274e 100644 --- a/src/app/[locale]/dashboard/logs/_components/error-details-dialog.test.tsx +++ b/src/app/[locale]/dashboard/logs/_components/error-details-dialog.test.tsx @@ -286,6 +286,7 @@ const messages = { }, logicTrace: { title: "Decision Chain", + singleRouteSelectionTitle: "Provider selection under single-route protection", noDecisionData: "No decision data", providersCount: "{count} providers", healthyCount: "{count} healthy", @@ -1188,6 +1189,7 @@ describe("error-details-dialog routing trace", () => { stickySlaMs: 20_000, racingTotalTimeoutMs: 60_000, stickyTimeoutCooldownMs: 300_000, + sessionTtlSeconds: 300, }, events: [ { @@ -1300,6 +1302,215 @@ describe("error-details-dialog routing trace", () => { expect(roundOne?.querySelector(".sm\\:grid-cols-2")).not.toBeNull(); expect(html).toContain("60000ms"); expect(html).toContain("300000ms"); + expect(html).toContain("300s"); + }); + + test("expands exact Discovery attempts with sanitized provider details and cancellation reasons", () => { + const longSecondError = + " second-attempt-429 " + "x".repeat(8_200) + "TAIL_NOT_RENDERED"; + const detailedTrace: RoutingTraceV1 = { + version: 1, + mode: "discovery", + startedAt: 1_000, + updatedAt: 5_000, + discoveryEnabled: true, + eligible: true, + events: [ + { + type: "attempt_started", + at: 1_000, + elapsedMs: 0, + round: 1, + attemptId: "80:1", + attemptKind: "normal", + provider: { id: 80, name: "same-provider", priority: 1 }, + }, + { + type: "attempt_finished", + at: 1_500, + elapsedMs: 500, + round: 1, + attemptId: "80:1", + attemptKind: "normal", + provider: { id: 80, name: "same-provider", priority: 1 }, + outcome: "failed", + statusCode: 403, + }, + { + type: "attempt_started", + at: 1_600, + elapsedMs: 600, + round: 1, + attemptId: "80:2", + attemptKind: "normal", + provider: { id: 80, name: "same-provider", priority: 1 }, + }, + { + type: "attempt_finished", + at: 2_000, + elapsedMs: 1_000, + round: 1, + attemptId: "80:2", + attemptKind: "normal", + provider: { id: 80, name: "same-provider", priority: 1 }, + outcome: "failed", + statusCode: 429, + }, + { + type: "attempt_started", + at: 2_100, + elapsedMs: 1_100, + round: 1, + attemptId: "91:3", + attemptKind: "normal", + provider: { id: 91, name: "legacy-provider", priority: 2 }, + }, + { + type: "attempt_finished", + at: 2_500, + elapsedMs: 1_500, + round: 1, + attemptId: "91:3", + attemptKind: "normal", + provider: { id: 91, name: "legacy-provider", priority: 2 }, + outcome: "failed", + statusCode: 503, + }, + { + type: "attempt_started", + at: 2_600, + elapsedMs: 1_600, + round: 1, + attemptId: "56:4", + attemptKind: "normal", + provider: { id: 56, name: "cancelled-provider", priority: 3 }, + }, + { + type: "attempt_finished", + at: 5_000, + elapsedMs: 4_000, + round: 1, + attemptId: "56:4", + attemptKind: "normal", + provider: { id: 56, name: "cancelled-provider", priority: 3 }, + outcome: "cancelled", + cancellationKind: "discovery_sla_timeout", + }, + ], + }; + const { container, unmount } = renderClientWithIntl( + + ); + + const attempts = Array.from( + container.querySelectorAll("[data-testid='discovery-attempt']") + ); + const secondAttempt = attempts.find((card) => card.textContent?.includes("HTTP 429")); + const secondToggle = + secondAttempt?.querySelector("[data-testid='discovery-attempt-toggle']") ?? null; + expect(secondToggle?.getAttribute("aria-expanded")).toBe("false"); + click(secondToggle); + expect(secondToggle?.getAttribute("aria-expanded")).toBe("true"); + expect(secondAttempt?.textContent).toContain("second-attempt-429"); + expect(secondAttempt?.textContent).not.toContain("first-attempt-403"); + expect(secondAttempt?.textContent).toContain("https://api.example.com/v2"); + expect(secondAttempt?.textContent).not.toContain("secret-two"); + expect(secondAttempt?.textContent).not.toContain("TAIL_NOT_RENDERED"); + expect(secondAttempt?.querySelector("script")).toBeNull(); + + const legacyAttempt = attempts.find((card) => card.textContent?.includes("HTTP 503")); + click(legacyAttempt?.querySelector("[data-testid='discovery-attempt-toggle']") ?? null); + expect(legacyAttempt?.textContent).toContain("third-attempt-503"); + expect(legacyAttempt?.textContent).toContain("https://api.example.com/v3"); + expect(legacyAttempt?.textContent).not.toContain("endpoint-user"); + expect(legacyAttempt?.textContent).not.toContain("endpoint-secret"); + expect(legacyAttempt?.textContent).not.toContain("abcdefghijklmnopqrstuvwxyz123456"); + expect(legacyAttempt?.textContent).not.toContain("bearer-secret-value"); + + const cancelledAttempt = attempts.find((card) => + card.textContent?.includes("cancelled-provider") + ); + click(cancelledAttempt?.querySelector("[data-testid='discovery-attempt-toggle']") ?? null); + expect(cancelledAttempt?.textContent).toContain("Discovery SLA expired"); + expect(cancelledAttempt?.textContent).not.toContain("Upstream error"); + unmount(); + }); + + test("labels a lease conflict as single-route protection while retaining provider selection", () => { + const protectedTrace: RoutingTraceV1 = { + version: 1, + mode: "single_upstream", + startedAt: 1_000, + updatedAt: 2_000, + discoveryEnabled: true, + eligible: false, + bypassReason: "lease_conflict", + events: [], + }; + const html = renderWithIntl( + + ); + + expect(html).toContain("Single-route protection"); + expect(html).toContain("Provider selection under single-route protection"); + expect(html).toContain("protected-provider"); }); test("shows a legacy mode and bypass reason while retaining the old chain", () => { diff --git a/src/app/[locale]/dashboard/logs/_components/error-details-dialog/components/DiscoveryTraceView.tsx b/src/app/[locale]/dashboard/logs/_components/error-details-dialog/components/DiscoveryTraceView.tsx index e5949ccd8..1c6439a71 100644 --- a/src/app/[locale]/dashboard/logs/_components/error-details-dialog/components/DiscoveryTraceView.tsx +++ b/src/app/[locale]/dashboard/logs/_components/error-details-dialog/components/DiscoveryTraceView.tsx @@ -2,6 +2,7 @@ import { CheckCircle, + ChevronDown, ChevronRight, CircleDot, Clock3, @@ -13,8 +14,12 @@ import { Zap, } from "lucide-react"; import { useTranslations } from "next-intl"; +import { useState } from "react"; import { Badge } from "@/components/ui/badge"; import { cn } from "@/lib/utils"; +import { redactJsonString } from "@/lib/utils/message-redaction"; +import { sanitizeErrorTextForDetail } from "@/lib/utils/upstream-error-detection"; +import type { ProviderChainItem } from "@/types/message"; import type { RoutingTraceV1 } from "@/types/routing-trace"; const KNOWN_BYPASS_REASONS = new Set([ @@ -42,6 +47,7 @@ type AttemptView = { id: string; providerId: number | null; providerName: string | null; + sequence: number | null; round: number; role: "sticky" | "normal" | "fallback"; priority: number | null; @@ -58,12 +64,17 @@ type AttemptView = { | "client_abort" | "deadline"; statusCode: number | null; + cancellationKind: string | null; + reason: string | null; fallbackPromoted: boolean; winnerCommitted: boolean; + chainItem: ProviderChainItem | null; history: Array<{ type: string; elapsedMs: number | null; outcome: AttemptView["outcome"] | null; + cancellationKind: string | null; + reason: string | null; }>; }; @@ -79,6 +90,80 @@ function asNumber(value: unknown): number | null { return typeof value === "number" && Number.isFinite(value) ? value : null; } +function parseAttemptSequence(attemptId: string): number | null { + const match = /:(\d+)$/.exec(attemptId); + return match ? Number(match[1]) : null; +} + +function truncateForDisplay(value: string, maxLength = 8_192): string { + return value.length > maxLength ? `${value.slice(0, maxLength)}\n...` : value; +} + +function sanitizeEndpoint(value: string | undefined): string | null { + if (!value) return null; + try { + const url = new URL(value); + url.username = ""; + url.password = ""; + url.search = ""; + url.hash = ""; + return truncateForDisplay(url.toString(), 2_048); + } catch { + return truncateForDisplay(value.split(/[?#]/, 1)[0], 2_048); + } +} + +function getChainErrorMessage(item: ProviderChainItem | null): string | null { + if (!item) return null; + const sanitize = (value: string) => + truncateForDisplay(sanitizeErrorTextForDetail(redactJsonString(value))); + if (item.errorMessage) return sanitize(item.errorMessage); + if (item.errorDetails?.provider?.upstreamBody) { + return sanitize(item.errorDetails.provider.upstreamBody); + } + if (item.errorDetails?.system?.errorMessage) { + return sanitize(item.errorDetails.system.errorMessage); + } + if (item.errorDetails?.clientError) return sanitize(item.errorDetails.clientError); + return null; +} + +function buildProviderChainLookup(providerChain: ProviderChainItem[]): { + byAttempt: Map; + uniqueByProvider: Map; +} { + const byAttempt = new Map(); + const byProvider = new Map(); + for (const item of providerChain) { + const items = byProvider.get(item.id) ?? []; + items.push(item); + byProvider.set(item.id, items); + if (Number.isFinite(item.attemptNumber)) { + byAttempt.set(`${item.id}:${item.attemptNumber}`, item); + } + } + + const uniqueByProvider = new Map(); + for (const [providerId, items] of byProvider) { + if (items.length === 1) uniqueByProvider.set(providerId, items[0]); + } + return { byAttempt, uniqueByProvider }; +} + +function findChainItem( + attemptId: string, + providerId: number | null, + lookup: ReturnType +): ProviderChainItem | null { + if (providerId == null) return null; + const sequence = parseAttemptSequence(attemptId); + if (sequence != null) { + const exact = lookup.byAttempt.get(`${providerId}:${sequence}`); + if (exact) return exact; + } + return lookup.uniqueByProvider.get(providerId) ?? null; +} + function eventType(event: TraceRecord): string { return asString(event.type) ?? asString(event.event) ?? "unknown"; } @@ -174,8 +259,9 @@ function applyEventOutcome(attempt: AttemptView, type: string, event: TraceRecor } } -function buildAttempts(trace: RoutingTraceV1): AttemptView[] { +function buildAttempts(trace: RoutingTraceV1, providerChain: ProviderChainItem[]): AttemptView[] { const attempts = new Map(); + const lookup = buildProviderChainLookup(providerChain); for (const rawEvent of trace.events) { const event = asRecord(rawEvent); @@ -194,6 +280,7 @@ function buildAttempts(trace: RoutingTraceV1): AttemptView[] { id: attemptId, providerId: provider.id, providerName: provider.name, + sequence: parseAttemptSequence(attemptId), round, role: normalizeRole(event.attemptKind ?? event.role ?? event.kind, round), priority: provider.priority, @@ -201,16 +288,23 @@ function buildAttempts(trace: RoutingTraceV1): AttemptView[] { elapsedMs: null, outcome: "pending", statusCode: null, + cancellationKind: null, + reason: null, fallbackPromoted: false, winnerCommitted: false, + chainItem: findChainItem(attemptId, provider.id, lookup), history: [], }; attempt.providerId ??= provider.id; attempt.providerName ??= provider.name; + attempt.sequence ??= parseAttemptSequence(attempt.id); + attempt.chainItem ??= findChainItem(attempt.id, attempt.providerId, lookup); attempt.round = Math.max(attempt.round, round); attempt.priority ??= provider.priority; attempt.statusCode ??= asNumber(event.statusCode); + attempt.cancellationKind ??= asString(event.cancellationKind); + attempt.reason ??= asString(event.reason); const elapsedMs = asNumber(event.elapsedMs); if (type === "attempt_started") attempt.startedAt = elapsedMs; if (elapsedMs != null) attempt.elapsedMs = elapsedMs; @@ -234,6 +328,8 @@ function buildAttempts(trace: RoutingTraceV1): AttemptView[] { type, elapsedMs, outcome: normalizeOutcome(event.outcome), + cancellationKind: asString(event.cancellationKind), + reason: asString(event.reason), }); } attempts.set(attemptId, attempt); @@ -355,8 +451,16 @@ function outcomeStyle(outcome: AttemptView["outcome"]): { export function RoutingModeBanner({ trace }: { trace: RoutingTraceV1 }) { const t = useTranslations("dashboard.logs.details.routingTrace"); + const isLeaseConflict = + trace.mode === "single_upstream" && trace.bypassReason === "lease_conflict"; const Icon = - trace.mode === "discovery" ? Zap : trace.mode === "single_upstream" ? Server : GitBranch; + trace.mode === "discovery" + ? Zap + : isLeaseConflict + ? ShieldCheck + : trace.mode === "single_upstream" + ? Server + : GitBranch; const rawReason = trace.bypassReason; const reason = rawReason && KNOWN_BYPASS_REASONS.has(rawReason) ? rawReason : rawReason ? "unknown" : null; @@ -367,7 +471,7 @@ export function RoutingModeBanner({ trace }: { trace: RoutingTraceV1 }) { {t("title")} - {t(`modes.${trace.mode}`)} + {isLeaseConflict ? t("modes.lease_conflict") : t(`modes.${trace.mode}`)}
{reason && ( @@ -379,9 +483,15 @@ export function RoutingModeBanner({ trace }: { trace: RoutingTraceV1 }) { ); } -export function DiscoveryTraceView({ trace }: { trace: RoutingTraceV1 }) { +export function DiscoveryTraceView({ + trace, + providerChain = [], +}: { + trace: RoutingTraceV1; + providerChain?: ProviderChainItem[]; +}) { const t = useTranslations("dashboard.logs.details.routingTrace"); - const attempts = buildAttempts(trace); + const attempts = buildAttempts(trace, providerChain); const grouped = new Map(); for (const attempt of attempts) { const group = grouped.get(attempt.round) ?? []; @@ -501,6 +611,12 @@ export function DiscoveryTraceView({ trace }: { trace: RoutingTraceV1 }) { value={`${numberFrom(config, "stickyTimeoutCooldownMs")}ms`} /> )} + {numberFrom(config, "sessionTtlSeconds") != null && ( + + )}
)} @@ -553,57 +669,137 @@ function TraceValue({ label, value }: { label: string; value: string | number | function AttemptCard({ attempt }: { attempt: AttemptView }) { const t = useTranslations("dashboard.logs.details.routingTrace"); + const [expanded, setExpanded] = useState(false); const style = outcomeStyle(attempt.outcome); const Icon = style.icon; const providerName = attempt.providerName ?? t("providerFallback", { id: attempt.providerId ?? "-" }); + const chainItem = attempt.chainItem; + const statusCode = chainItem?.statusCode ?? attempt.statusCode; + const endpoint = sanitizeEndpoint(chainItem?.endpointUrl); + const errorMessage = getChainErrorMessage(chainItem); + const knownCancellationKinds = new Set([ + "discovery_loser", + "discovery_sla_timeout", + "round_timeout", + "sticky_timeout", + "request_deadline", + "client_abort", + "winner_committed", + ]); + const cancellationLabel = attempt.cancellationKind + ? knownCancellationKinds.has(attempt.cancellationKind) + ? t(`cancellationKinds.${attempt.cancellationKind}`) + : attempt.cancellationKind + : null; return (
-
- -
-
- - {providerName} - - {attempt.elapsedMs != null && ( - - {t("elapsed", { elapsed: Math.max(0, Math.round(attempt.elapsedMs)) })} + + {expanded && ( +
+
+ {attempt.providerId != null && ( + )} - {attempt.statusCode != null && ( - HTTP {attempt.statusCode} + {attempt.sequence != null && ( + + )} + {statusCode != null && ( + )}
- {(attempt.fallbackPromoted || attempt.winnerCommitted) && ( -
- - {attempt.winnerCommitted ? t("winnerCommitted") : t("fallbackPromoted")} + {endpoint && ( +
+
{t("attemptDetails.endpoint")}
+ {endpoint} +
+ )} + {errorMessage && ( +
+
+ {t("attemptDetails.error")} +
+ 
+                {errorMessage}
+
+
+ )} + {cancellationLabel && ( +
+ {t("attemptDetails.cancellation")}:{" "} + {cancellationLabel}
)} {attempt.history.length > 0 && ( -
+
+
{t("attemptDetails.timeline")}
{attempt.history.map((event, index) => (
@@ -619,7 +815,7 @@ function AttemptCard({ attempt }: { attempt: AttemptView }) {
)}
-
+ )}
); } diff --git a/src/app/[locale]/dashboard/logs/_components/error-details-dialog/components/LogicTraceTab.tsx b/src/app/[locale]/dashboard/logs/_components/error-details-dialog/components/LogicTraceTab.tsx index 6985debb5..22bc1263e 100644 --- a/src/app/[locale]/dashboard/logs/_components/error-details-dialog/components/LogicTraceTab.tsx +++ b/src/app/[locale]/dashboard/logs/_components/error-details-dialog/components/LogicTraceTab.tsx @@ -15,6 +15,7 @@ import { Link2, RefreshCw, Server, + ShieldCheck, XCircle, Zap, } from "lucide-react"; @@ -188,6 +189,9 @@ export function LogicTraceTab({ const totalProviders = decisionContext?.totalProviders || 0; const afterHealthCheck = decisionContext?.afterHealthCheck || 0; const normalizedRoutingTrace = normalizeRoutingTrace(routingTrace); + const isLeaseConflictProtection = + normalizedRoutingTrace?.mode === "single_upstream" && + normalizedRoutingTrace.bypassReason === "lease_conflict"; // Calculate step offset for session reuse flow const sessionReuseStepOffset = isSessionReuseFlow ? 1 : 0; @@ -196,7 +200,7 @@ export function LogicTraceTab({ return (
- +
); } @@ -263,12 +267,16 @@ export function LogicTraceTab({

- {isSessionReuseFlow ? ( + {isLeaseConflictProtection ? ( + + ) : isSessionReuseFlow ? ( ) : ( )} - {t("logicTrace.title")} + {isLeaseConflictProtection + ? t("logicTrace.singleRouteSelectionTitle") + : t("logicTrace.title")}

{isSessionReuseFlow ? ( diff --git a/src/app/[locale]/dashboard/logs/_components/provider-chain-popover.test.tsx b/src/app/[locale]/dashboard/logs/_components/provider-chain-popover.test.tsx index 7906ff832..421936d9f 100644 --- a/src/app/[locale]/dashboard/logs/_components/provider-chain-popover.test.tsx +++ b/src/app/[locale]/dashboard/logs/_components/provider-chain-popover.test.tsx @@ -658,4 +658,88 @@ describe("provider-chain-popover Discovery summary", () => { expect(html).toContain("Note: payload may have been forwarded"); expect(html).toContain("Why CCH cannot retry this response on the server"); }); + + test("marks lease-conflict single-upstream routing without hiding selector priority", () => { + const routingTrace: RoutingTraceV1 = { + version: 1, + mode: "single_upstream", + startedAt: 1_000, + updatedAt: 2_000, + discoveryEnabled: true, + eligible: false, + bypassReason: "lease_conflict", + events: [], + }; + const html = renderWithIntl( + + ); + + expect(html).toContain("Single-route protection"); + expect(html).toContain("Another request owns the Discovery lease"); + expect(html).toContain("P1"); + expect(html).toContain("lucide-shield-check"); + }); + + test("keeps lease-conflict protection visible after serial provider fallback", () => { + const routingTrace: RoutingTraceV1 = { + version: 1, + mode: "single_upstream", + startedAt: 1_000, + updatedAt: 3_000, + discoveryEnabled: true, + eligible: false, + bypassReason: "lease_conflict", + events: [], + }; + const html = renderWithIntl( + + ); + + expect(html).toContain("Single-route protection"); + expect(html).toContain("lucide-shield-check"); + expect(html).toContain("primary"); + expect(html).toContain("backup"); + }); }); diff --git a/src/app/[locale]/dashboard/logs/_components/provider-chain-popover.tsx b/src/app/[locale]/dashboard/logs/_components/provider-chain-popover.tsx index af08d4a4e..0023c669c 100644 --- a/src/app/[locale]/dashboard/logs/_components/provider-chain-popover.tsx +++ b/src/app/[locale]/dashboard/logs/_components/provider-chain-popover.tsx @@ -10,6 +10,7 @@ import { Link2, MinusCircle, RefreshCw, + ShieldCheck, XCircle, Zap, } from "lucide-react"; @@ -331,6 +332,9 @@ export function ProviderChainPopover({ // Determine max width based on whether cost badge is present const maxWidthClass = hasCostBadge ? "max-w-[140px]" : "max-w-[180px]"; + const isLeaseConflictProtection = + normalizedRoutingTrace?.mode === "single_upstream" && + normalizedRoutingTrace.bypassReason === "lease_conflict"; // Check if this is a session reuse const isSessionReuse = @@ -354,22 +358,41 @@ export function ProviderChainPopover({ - + + {isLeaseConflictProtection && ( +
{/* Provider name */}
{displayName}
+ {isLeaseConflictProtection && ( +
+
+ )} {singleRequestItem?.statusCode && (
{/* Request count badge */} - {isHedge ? ( + {isLeaseConflictProtection ? ( +