diff --git a/package.json b/package.json index 786c92e13..bc60d55ca 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "test:coverage:proxy-guard-pipeline": "vitest run --config tests/configs/proxy-guard-pipeline.config.ts --coverage", "test:coverage:include-session-id-in-errors": "vitest run --config tests/configs/include-session-id-in-errors.config.ts --coverage", "test:coverage:usage-logs-sessionid-search": "vitest run --config tests/configs/usage-logs-sessionid-search.config.ts --coverage", + "test:coverage:session-binding": "vitest run --config tests/configs/session-binding.config.ts --coverage", "test:ci": "vitest run --reporter=default --reporter=junit --outputFile.junit=reports/vitest-junit.xml", "test:v1": "vitest run --config tests/configs/v1.config.ts --coverage --reporter=verbose && bun scripts/check-v1-critical-coverage.ts", "openapi:generate": "bun scripts/generate-v1-types.ts", diff --git a/src/app/v1/_lib/proxy/forwarder.ts b/src/app/v1/_lib/proxy/forwarder.ts index b0d60e50f..b93c9c7bf 100644 --- a/src/app/v1/_lib/proxy/forwarder.ts +++ b/src/app/v1/_lib/proxy/forwarder.ts @@ -100,7 +100,10 @@ import { import { ProxyProviderResolver } from "./provider-selector"; import { finalizeHedgeLoserBilling } from "./response-handler"; import type { ProxySession } from "./session"; -import { setDeferredStreamingFinalization } from "./stream-finalization"; +import { + type DeferredStreamingHedgeBindingAuthority, + setDeferredStreamingFinalization, +} from "./stream-finalization"; import { detectThinkingBudgetRectifierTrigger, rectifyThinkingBudget, @@ -4571,8 +4574,12 @@ export class ProxyForwarder { abortAllAttempts(attempt, "hedge_loser"); - if (session.sessionId) { - void (async () => { + // A non-hedged request is finalized through response-handler. Updating + // here as well would perform a duplicate binding read/CAS before the + // stream has passed its final validation. + let hedgeBindingAuthorityPromise: Promise | undefined; + if (session.sessionId && isActualHedgeWin) { + hedgeBindingAuthorityPromise = (async () => { const bindingResult = await SessionManager.updateSessionBindingSmart( session.sessionId!, attempt.provider.id, @@ -4595,15 +4602,31 @@ export class ProxyForwarder { } if (session.shouldTrackSessionObservability()) { - await SessionManager.updateSessionProvider(session.sessionId!, { + void SessionManager.updateSessionProvider(session.sessionId!, { providerId: attempt.provider.id, providerName: attempt.provider.name, + }).catch((observabilityError) => { + logger.error( + "ProxyForwarder: Failed to update observable session provider for hedge winner", + { error: observabilityError } + ); }); } + + return { + // Only the exact snapshot returned by the first-byte CAS may keep + // a versioned binding alive or clear it after a failed stream. + snapshot: bindingResult.bindingSnapshot ?? null, + // A generic clear is safe only when this request demonstrably + // committed the legacy binding. A versioned CAS conflict must not + // clear a newer generation that happens to use the same Provider. + legacyClearAllowed: bindingResult.legacyBindingUpdated === true, + }; })().catch((bindingError) => { logger.error("ProxyForwarder: Failed to update session provider info for hedge winner", { error: bindingError, }); + return { snapshot: null, legacyClearAllowed: false }; }); } @@ -4620,6 +4643,7 @@ export class ProxyForwarder { upstreamStatusCode: attempt.response.status, isHedgeWinner: isActualHedgeWin, billHedgeLosers, + hedgeBindingAuthorityPromise, }); const response = new Response( @@ -5011,16 +5035,17 @@ export class ProxyForwarder { expectedProviderId: number | null ): Promise { if (!session.sessionId) return; - await SessionManager.clearSessionProvider(session.sessionId, expectedProviderId); + const keyId = session.authState?.key?.id ?? session.messageContext?.key?.id ?? null; + await SessionManager.clearSessionProvider(session.sessionId, expectedProviderId, keyId); } private static async clearSessionProviderBindings( session: ProxySession, expectedProviderIds: Iterable ): Promise { - for (const providerId of new Set(expectedProviderIds)) { - await ProxyForwarder.clearSessionProviderBinding(session, providerId); - } + if (!session.sessionId) return; + const keyId = session.authState?.key?.id ?? session.messageContext?.key?.id ?? null; + await SessionManager.clearSessionProviders(session.sessionId, expectedProviderIds, keyId); } private static markProviderFailed( diff --git a/src/app/v1/_lib/proxy/provider-selector.ts b/src/app/v1/_lib/proxy/provider-selector.ts index 90224ccf5..e81d32593 100644 --- a/src/app/v1/_lib/proxy/provider-selector.ts +++ b/src/app/v1/_lib/proxy/provider-selector.ts @@ -492,10 +492,8 @@ export class ProxyProviderResolver { } // 从 Redis 读取该 session 绑定的 provider - const providerId = await SessionManager.getSessionProvider( - session.sessionId, - session.authState?.key?.id ?? null - ); + const keyId = session.authState?.key?.id ?? session.messageContext?.key?.id ?? null; + const providerId = await SessionManager.getSessionProvider(session.sessionId, keyId); if (!providerId) { logger.debug("ProviderSelector: Session has no bound provider", { sessionId: session.sessionId, @@ -510,7 +508,7 @@ export class ProxyProviderResolver { sessionId: session.sessionId, providerId, }); - await SessionManager.clearSessionProvider(session.sessionId, providerId); + await SessionManager.clearSessionProvider(session.sessionId, providerId, keyId); return null; } @@ -520,7 +518,7 @@ export class ProxyProviderResolver { providerId: provider.id, providerName: provider.name, }); - await SessionManager.clearSessionProvider(session.sessionId, providerId); + await SessionManager.clearSessionProvider(session.sessionId, providerId, keyId); return null; } @@ -534,7 +532,7 @@ export class ProxyProviderResolver { activeTimeEnd: provider.activeTimeEnd, timezone: systemTimezone, }); - await SessionManager.clearSessionProvider(session.sessionId, providerId); + await SessionManager.clearSessionProvider(session.sessionId, providerId, keyId); return null; } @@ -575,7 +573,7 @@ export class ProxyProviderResolver { providerType: provider.providerType, originalFormat: session.originalFormat, }); - await SessionManager.clearSessionProvider(session.sessionId, providerId); + await SessionManager.clearSessionProvider(session.sessionId, providerId, keyId); return null; } @@ -594,7 +592,7 @@ export class ProxyProviderResolver { // 清除过时绑定，避免 SET NX 死锁 // 当 session 内请求模型发生变化时，旧绑定已无意义， // 清除后新的成功请求可通过 SET NX 重新绑定匹配的 provider - await SessionManager.clearSessionProvider(session.sessionId, providerId); + await SessionManager.clearSessionProvider(session.sessionId, providerId, keyId); logger.info("ProviderSelector: Cleared stale provider binding (model mismatch)", { sessionId: session.sessionId, staleProviderId: provider.id, @@ -650,7 +648,7 @@ export class ProxyProviderResolver { ], }, }); - await SessionManager.clearSessionProvider(session.sessionId, providerId); + await SessionManager.clearSessionProvider(session.sessionId, providerId, keyId); return null; } diff --git a/src/app/v1/_lib/proxy/response-handler.ts b/src/app/v1/_lib/proxy/response-handler.ts index d633c8cbd..d5a522a66 100644 --- a/src/app/v1/_lib/proxy/response-handler.ts +++ b/src/app/v1/_lib/proxy/response-handler.ts @@ -58,6 +58,7 @@ import { isClientAbortError, isTransportError } from "./errors"; import type { ProxySession } from "./session"; import { consumeDeferredStreamingFinalization, + type DeferredStreamingBindingHeartbeat, peekDeferredStreamingFinalization, } from "./stream-finalization"; @@ -97,6 +98,96 @@ const STREAM_FINALIZATION_MAX_MS = 120_000; const STREAM_FAILURE_PERSISTENCE_MAX_MS = 5_000; const NON_STREAM_TERMINAL_PERSISTENCE_ERROR = Symbol("non_stream_terminal_persistence_error"); +function startHedgeBindingHeartbeat(session: ProxySession): void { + const deferred = peekDeferredStreamingFinalization(session); + const authorityPromise = deferred?.hedgeBindingAuthorityPromise; + if (!deferred?.isHedgeWinner || !authorityPromise || deferred.hedgeBindingHeartbeat) return; + + let periodicActive = true; + let authorityLost = false; + let timer: ReturnType | null = null; + let touchInFlight: Promise | null = null; + let completionPromise: Promise | null = null; + + const stopPeriodic = () => { + periodicActive = false; + if (timer) { + clearInterval(timer); + timer = null; + } + }; + + const loseAuthority = (status: string, reason?: string) => { + if (authorityLost) return; + authorityLost = true; + stopPeriodic(); + logger.warn("[ResponseHandler] Hedge binding heartbeat stopped", { + sessionId: session.sessionId, + status, + reason, + }); + }; + + const touch = (allowAfterStop = false): Promise => { + if (authorityLost || (!periodicActive && !allowAfterStop)) return Promise.resolve(false); + if (touchInFlight) return touchInFlight; + + const operation = (async () => { + const { snapshot } = await authorityPromise; + if (!snapshot) { + authorityLost = true; + stopPeriodic(); + return false; + } + if (authorityLost || (!periodicActive && !allowAfterStop)) return false; + + const touched = await SessionManager.touchVersionedSessionBinding(snapshot); + if ( + touched.status !== "ok" || + touched.snapshot.generation !== snapshot.generation || + touched.snapshot.providerId !== snapshot.providerId + ) { + loseAuthority(touched.status, "reason" in touched ? touched.reason : "snapshot_mismatch"); + return false; + } + return true; + })() + .catch((error) => { + loseAuthority("error", error instanceof Error ? error.message : String(error)); + return false; + }) + .finally(() => { + if (touchInFlight === operation) touchInFlight = null; + }); + touchInFlight = operation; + return operation; + }; + + const lifecycle: DeferredStreamingBindingHeartbeat = { + stop: stopPeriodic, + complete: () => { + if (completionPromise) return completionPromise; + stopPeriodic(); + completionPromise = (async () => { + if (touchInFlight) await touchInFlight; + if (authorityLost) return; + await touch(true); + })(); + return completionPromise; + }, + }; + deferred.hedgeBindingHeartbeat = lifecycle; + + // The first touch validates that ownership really transferred with the + // first-byte CAS. Subsequent touches keep streams longer than SESSION_TTL alive. + void touch(); + const intervalMs = Math.max(250, SessionManager.getVersionedSessionBindingRefreshIntervalMs()); + timer = setInterval(() => { + void touch(); + }, intervalMs); + timer.unref?.(); +} + type MessageRequestTerminalDetails = Parameters[1]; type NonStreamTerminalPersistenceError = Error & { [NON_STREAM_TERMINAL_PERSISTENCE_ERROR]: true; @@ -1149,7 +1240,29 @@ function finalizeDeferredStreamingFinalizationIfNeeded( const providerIdForPersistence = meta?.providerId ?? provider?.id ?? null; const clearSessionBinding = async () => { if (!session.sessionId) return; - await SessionManager.clearSessionProvider(session.sessionId, providerIdForPersistence); + const hedgeAuthority = meta?.isHedgeWinner + ? await meta.hedgeBindingAuthorityPromise + : undefined; + if (hedgeAuthority?.snapshot) { + const hedgeSnapshot = hedgeAuthority.snapshot; + const cleared = await SessionManager.clearVersionedSessionProvider( + hedgeSnapshot, + providerIdForPersistence + ); + if (cleared.status !== "ok") { + logger.warn("[ResponseHandler] Hedge winner binding clear stopped", { + sessionId: hedgeSnapshot.sessionId, + providerId: providerIdForPersistence, + reason: cleared.reason, + }); + } + return; + } + if (meta?.isHedgeWinner && !hedgeAuthority?.legacyClearAllowed) { + return; + } + const keyId = session.authState?.key?.id ?? session.messageContext?.key?.id ?? null; + await SessionManager.clearSessionProvider(session.sessionId, providerIdForPersistence, keyId); }; const isHedgeWinner = meta?.isHedgeWinner === true; @@ -1239,11 +1352,15 @@ function finalizeDeferredStreamingFinalizationIfNeeded( ((clientAborted || !streamEndedNormally) && !clientAbortCompleteSuccess) || detected.isError || (upstreamStatusCode >= 400 && errorMessage !== null); + if (shouldClearSessionBindingOnFailure) { + meta?.hedgeBindingHeartbeat?.stop(); + } // 未启用延迟结算 / provider 缺失： // - 只返回“内部状态码 + 错误原因”，由调用方写入统计； // - 不在这里更新熔断/绑定（meta 缺失意味着 Forwarder 没有启用延迟结算；provider 缺失意味着无法归因）。 if (!meta || !provider) { + meta?.hedgeBindingHeartbeat?.stop(); return { effectiveStatusCode, errorMessage, @@ -1461,7 +1578,13 @@ function finalizeDeferredStreamingFinalizationIfNeeded( }); } + // Stop periodic refresh at the stream boundary and issue one final + // generation-safe touch so the next turn receives a full binding TTL. + // complete() is idempotent and permanently stops after any authority conflict. + const hedgeBindingCompletion = meta.hedgeBindingHeartbeat?.complete(); const commitSideEffects = async () => { + await hedgeBindingCompletion; + if (meta.endpointId != null) { try { const { recordEndpointSuccess } = await import("@/lib/endpoint-circuit-breaker"); @@ -1806,7 +1929,8 @@ export class ProxyResponseHandler { if (session.sessionId) { const sessionId = session.sessionId; postTerminalSideEffects.push(async () => { - await SessionManager.clearSessionProvider(sessionId, provider.id); + const keyId = session.authState?.key?.id ?? session.messageContext?.key?.id ?? null; + await SessionManager.clearSessionProvider(sessionId, provider.id, keyId); }); } if ( @@ -1976,7 +2100,8 @@ export class ProxyResponseHandler { if (session.sessionId) { const sessionId = session.sessionId; postTerminalSideEffects.push(async () => { - await SessionManager.clearSessionProvider(sessionId, provider.id); + const keyId = session.authState?.key?.id ?? session.messageContext?.key?.id ?? null; + await SessionManager.clearSessionProvider(sessionId, provider.id, keyId); const sessionUsagePayload: SessionUsageUpdate = { status: @@ -2533,6 +2658,8 @@ export class ProxyResponseHandler { return response; } + startHedgeBindingHeartbeat(session); + let processedStream: ReadableStream = response.body; // --- GEMINI STREAM HANDLING --- diff --git a/src/app/v1/_lib/proxy/stream-finalization.ts b/src/app/v1/_lib/proxy/stream-finalization.ts index 0f989bb3b..c1eafe42b 100644 --- a/src/app/v1/_lib/proxy/stream-finalization.ts +++ b/src/app/v1/_lib/proxy/stream-finalization.ts @@ -1,5 +1,18 @@ +import type { SessionBindingSnapshot } from "@/lib/redis/session-binding"; import type { ProxySession } from "./session"; +export type DeferredStreamingBindingHeartbeat = { + stop: () => void; + complete: () => Promise; +}; + +export type DeferredStreamingHedgeBindingAuthority = { + /** Exact versioned generation written by the first-byte winner, when available. */ + snapshot: SessionBindingSnapshot | null; + /** Only a confirmed successful legacy write may use the non-versioned clear path. */ + legacyClearAllowed: boolean; +}; + /** * 流式响应（SSE）在“收到响应头”时无法确定成功与否： * - 上游可能返回 HTTP 200，但 body 是错误 JSON（假 200） @@ -35,6 +48,10 @@ export type DeferredStreamingFinalization = { * coexists with asynchronously accumulated loser costs without clobbering. */ billHedgeLosers?: boolean; + /** Binding authority established by the legacy Hedge winner's first-byte write. */ + hedgeBindingAuthorityPromise?: Promise; + /** ResponseHandler-owned runtime lifecycle; attached when streaming starts. */ + hedgeBindingHeartbeat?: DeferredStreamingBindingHeartbeat; }; const deferredMeta = new WeakMap(); diff --git a/src/lib/redis/client.ts b/src/lib/redis/client.ts index 21bdc38dc..ae827d95e 100644 --- a/src/lib/redis/client.ts +++ b/src/lib/redis/client.ts @@ -3,6 +3,7 @@ import { getEnvConfig } from "@/lib/config/env.schema"; import { logger } from "@/lib/logger"; let redisClient: Redis | null = null; +let redisClientUrl: string | null = null; function maskRedisUrl(redisUrl: string) { try { @@ -105,9 +106,20 @@ export function getRedisClient(input?: { allowWhenRateLimitDisabled?: boolean }) const safeRedisUrl = maskRedisUrl(redisUrl); + if (redisClient && redisClientUrl !== redisUrl) { + const staleClient = redisClient; + redisClient = null; + redisClientUrl = null; + staleClient.disconnect(); + logger.warn("[Redis] Connection configuration changed, replacing client", { + redisUrl: safeRedisUrl, + }); + } + if (redisClient) { if (redisClient.status === "end") { redisClient = null; + redisClientUrl = null; } else { return redisClient; } @@ -123,6 +135,7 @@ export function getRedisClient(input?: { allowWhenRateLimitDisabled?: boolean }) // 3. 使用组合后的配置创建客户端 const client = new Redis(redisUrl, redisOptions); redisClient = client; + redisClientUrl = redisUrl; // 4. 保持原始的事件监听器 client.on("connect", () => { @@ -153,6 +166,7 @@ export function getRedisClient(input?: { allowWhenRateLimitDisabled?: boolean }) if (redisClient !== client) return; logger.warn("[Redis] Connection ended, resetting client", { redisUrl: safeRedisUrl }); redisClient = null; + redisClientUrl = null; }); // 5. 返回客户端实例 @@ -179,6 +193,7 @@ export async function closeRedis(): Promise { } finally { if (redisClient === client) { redisClient = null; + redisClientUrl = null; } } } diff --git a/src/lib/redis/lua-scripts.ts b/src/lib/redis/lua-scripts.ts index c0da78e89..6b4a9829b 100644 --- a/src/lib/redis/lua-scripts.ts +++ b/src/lib/redis/lua-scripts.ts @@ -4,6 +4,31 @@ * 用于保证 Redis 操作的原子性 */ +/** + * Delete a legacy provider mirror only when it still contains the value that + * the guarded fallback mutation wrote. This is intentionally single-key so it + * remains usable on Redis Cluster when the multi-key binding scripts are not. + */ +export const DELETE_LEGACY_PROVIDER_IF_VALUE = ` +if redis.call('GET', KEYS[1]) == ARGV[1] then + return redis.call('DEL', KEYS[1]) +end +return 0 +`; + +/** + * Restore a legacy provider mirror only when the guarded fallback clear left + * it absent. The conditional write keeps a concurrent versioned writer's + * newer mirror value intact. + */ +export const RESTORE_LEGACY_PROVIDER_IF_ABSENT = ` +if redis.call('EXISTS', KEYS[1]) == 0 then + redis.call('SETEX', KEYS[1], ARGV[2], ARGV[1]) + return 1 +end +return 0 +`; + /** * Atomic concurrency check + session tracking (TC-041 fixed version) * @@ -376,3 +401,504 @@ end return tostring(total) `; + +/** + * Atomically read a tenant-scoped session binding and reconcile it with the + * legacy session-only mirror during a rolling upgrade. + * + * KEYS[1]: canonical binding hash + * KEYS[2]: legacy provider string + * KEYS[3]: legacy key owner string + * ARGV[1]: current key id + * ARGV[2]: generation to use when initializing/upgrading + * ARGV[3]: binding TTL in seconds + * + * Return: + * - {"ok", source, generation, providerIdOrEmpty} + * - {"conflict", reason} + */ +export const READ_OR_RECONCILE_SESSION_BINDING = ` +local binding_key = KEYS[1] +local legacy_provider_key = KEYS[2] +local legacy_owner_key = KEYS[3] + +local current_key_id = ARGV[1] +local new_generation = ARGV[2] +local ttl = tonumber(ARGV[3]) + +local function is_positive_integer(value) + local parsed = tonumber(value) + return parsed and parsed > 0 and parsed == math.floor(parsed) +end + +if not ttl or ttl <= 0 or current_key_id == '' or new_generation == '' then + return {'conflict', 'invalid_input'} +end + +local legacy_provider = redis.call('GET', legacy_provider_key) +local legacy_owner = redis.call('GET', legacy_owner_key) +local binding_exists = redis.call('EXISTS', binding_key) == 1 + +if binding_exists then + local binding = redis.call('HMGET', binding_key, 'key_id', 'generation', 'provider_id') + local binding_key_id = binding[1] + local generation = binding[2] + local provider_id = binding[3] + + if not binding_key_id or not generation or binding_key_id == '' or generation == '' then + return {'conflict', 'canonical_corrupt'} + end + if binding_key_id ~= current_key_id then + return {'conflict', 'canonical_key_mismatch'} + end + if not legacy_owner then + return {'conflict', 'mirror_missing'} + end + if legacy_owner ~= current_key_id then + return {'conflict', 'foreign_legacy_owner'} + end + + if provider_id then + if not is_positive_integer(provider_id) then + return {'conflict', 'canonical_corrupt'} + end + if legacy_provider ~= provider_id then + return {'conflict', 'mirror_conflict'} + end + redis.call('EXPIRE', legacy_provider_key, ttl) + elseif legacy_provider then + return {'conflict', 'mirror_conflict'} + end + + redis.call('EXPIRE', binding_key, ttl) + redis.call('EXPIRE', legacy_owner_key, ttl) + return {'ok', 'existing', generation, provider_id or ''} +end + +if legacy_owner and legacy_owner ~= current_key_id then + return {'conflict', 'foreign_legacy_owner'} +end +if legacy_provider and not legacy_owner then + return {'conflict', 'orphan_legacy_provider'} +end +if legacy_provider and not is_positive_integer(legacy_provider) then + return {'conflict', 'invalid_legacy_provider'} +end + +if not legacy_owner and not legacy_provider then + redis.call('HSET', binding_key, 'key_id', current_key_id, 'generation', new_generation) + redis.call('HDEL', binding_key, 'provider_id') + redis.call('EXPIRE', binding_key, ttl) + redis.call('SETEX', legacy_owner_key, ttl, current_key_id) + return {'ok', 'created', new_generation, ''} +end + +-- At this point the legacy owner is current_key_id. The provider may be absent, +-- which is the valid null-binding mirror used by a fresh session. +redis.call('HSET', binding_key, 'key_id', current_key_id, 'generation', new_generation) +if legacy_provider then + redis.call('HSET', binding_key, 'provider_id', legacy_provider) + redis.call('EXPIRE', legacy_provider_key, ttl) +else + redis.call('HDEL', binding_key, 'provider_id') +end +redis.call('EXPIRE', binding_key, ttl) +redis.call('EXPIRE', legacy_owner_key, ttl) +return {'ok', 'legacy_upgraded', new_generation, legacy_provider or ''} +`; + +/** + * Compare-and-set a provider on an existing versioned session binding. + * The canonical hash and both legacy mirrors are validated before dual-write. + * Missing canonical state is always a conflict and is never initialized here. + * + * KEYS[1]: canonical binding hash + * KEYS[2]: legacy provider string + * KEYS[3]: legacy key owner string + * ARGV[1]: current key id + * ARGV[2]: expected generation + * ARGV[3]: next generation + * ARGV[4]: next provider id + * ARGV[5]: binding TTL in seconds + */ +export const CAS_SESSION_BINDING = ` +local binding_key = KEYS[1] +local legacy_provider_key = KEYS[2] +local legacy_owner_key = KEYS[3] + +local current_key_id = ARGV[1] +local expected_generation = ARGV[2] +local next_generation = ARGV[3] +local next_provider_id = ARGV[4] +local ttl = tonumber(ARGV[5]) + +local function is_positive_integer(value) + local parsed = tonumber(value) + return parsed and parsed > 0 and parsed == math.floor(parsed) +end + +if not ttl or ttl <= 0 or current_key_id == '' or expected_generation == '' or + next_generation == '' or not is_positive_integer(next_provider_id) then + return {'conflict', 'invalid_input'} +end +if redis.call('EXISTS', binding_key) == 0 then + return {'conflict', 'canonical_missing'} +end + +local binding = redis.call('HMGET', binding_key, 'key_id', 'generation', 'provider_id') +local binding_key_id = binding[1] +local generation = binding[2] +local current_provider_id = binding[3] + +if not binding_key_id or not generation or binding_key_id == '' or generation == '' then + return {'conflict', 'canonical_corrupt'} +end +if binding_key_id ~= current_key_id then + return {'conflict', 'canonical_key_mismatch'} +end +if generation ~= expected_generation then + return {'conflict', 'generation_mismatch'} +end + +local legacy_owner = redis.call('GET', legacy_owner_key) +local legacy_provider = redis.call('GET', legacy_provider_key) +if current_provider_id and not is_positive_integer(current_provider_id) then + return {'conflict', 'canonical_corrupt'} +end +if legacy_provider and not is_positive_integer(legacy_provider) then + return {'conflict', 'invalid_legacy_provider'} +end +if not legacy_owner then + return {'conflict', 'mirror_missing'} +end +if legacy_owner ~= current_key_id then + return {'conflict', 'foreign_legacy_owner'} +end +if current_provider_id then + if legacy_provider ~= current_provider_id then + return {'conflict', 'mirror_conflict'} + end +elseif legacy_provider then + return {'conflict', 'mirror_conflict'} +end + +redis.call('HSET', binding_key, + 'key_id', current_key_id, + 'generation', next_generation, + 'provider_id', next_provider_id) +redis.call('EXPIRE', binding_key, ttl) +redis.call('SETEX', legacy_owner_key, ttl, current_key_id) +redis.call('SETEX', legacy_provider_key, ttl, next_provider_id) +return {'ok', 'updated', next_generation, next_provider_id} +`; + +/** + * Extend an existing binding's TTL only while the complete captured snapshot + * still matches canonical state and both legacy mirrors. This operation never + * initializes a missing binding or rotates its generation. + * + * KEYS[1]: canonical binding hash + * KEYS[2]: legacy provider string + * KEYS[3]: legacy key owner string + * ARGV[1]: current key id + * ARGV[2]: expected generation + * ARGV[3]: expected provider id, or empty for a null binding + * ARGV[4]: binding TTL in seconds + */ +export const TOUCH_SESSION_BINDING = ` +local binding_key = KEYS[1] +local legacy_provider_key = KEYS[2] +local legacy_owner_key = KEYS[3] + +local current_key_id = ARGV[1] +local expected_generation = ARGV[2] +local expected_provider_id = ARGV[3] +local ttl = tonumber(ARGV[4]) + +local function is_positive_integer(value) + local parsed = tonumber(value) + return parsed and parsed > 0 and parsed == math.floor(parsed) +end + +if not ttl or ttl <= 0 or current_key_id == '' or expected_generation == '' or + (expected_provider_id ~= '' and not is_positive_integer(expected_provider_id)) then + return {'conflict', 'invalid_input'} +end +if redis.call('EXISTS', binding_key) == 0 then + return {'conflict', 'canonical_missing'} +end + +local binding = redis.call('HMGET', binding_key, 'key_id', 'generation', 'provider_id') +local binding_key_id = binding[1] +local generation = binding[2] +local current_provider_id = binding[3] + +if not binding_key_id or not generation or binding_key_id == '' or generation == '' then + return {'conflict', 'canonical_corrupt'} +end +if binding_key_id ~= current_key_id then + return {'conflict', 'canonical_key_mismatch'} +end +if current_provider_id and not is_positive_integer(current_provider_id) then + return {'conflict', 'canonical_corrupt'} +end +if generation ~= expected_generation then + return {'conflict', 'generation_mismatch'} +end +if (current_provider_id or '') ~= expected_provider_id then + return {'conflict', 'provider_mismatch'} +end + +local legacy_owner = redis.call('GET', legacy_owner_key) +local legacy_provider = redis.call('GET', legacy_provider_key) +if legacy_provider and not is_positive_integer(legacy_provider) then + return {'conflict', 'invalid_legacy_provider'} +end +if not legacy_owner then + return {'conflict', 'mirror_missing'} +end +if legacy_owner ~= current_key_id then + return {'conflict', 'foreign_legacy_owner'} +end +if current_provider_id then + if legacy_provider ~= current_provider_id then + return {'conflict', 'mirror_conflict'} + end +elseif legacy_provider then + return {'conflict', 'mirror_conflict'} +end + +redis.call('EXPIRE', binding_key, ttl) +redis.call('EXPIRE', legacy_owner_key, ttl) +if current_provider_id then + redis.call('EXPIRE', legacy_provider_key, ttl) +end +return {'ok', 'touched', generation, current_provider_id or ''} +`; + +/** + * Compare-and-clear an existing versioned session binding. A cooldown marker + * may be written in the same transaction when clearing a timed-out provider. + * + * KEYS[1]: canonical binding hash + * KEYS[2]: legacy provider string + * KEYS[3]: legacy key owner string + * KEYS[4]: tenant-scoped cooldown key (unused when cooldown TTL is zero) + * ARGV[1]: current key id + * ARGV[2]: expected generation + * ARGV[3]: next generation + * ARGV[4]: expected provider id, or empty for a null binding + * ARGV[5]: binding TTL in seconds + * ARGV[6]: cooldown provider id, or empty + * ARGV[7]: cooldown TTL in seconds, or zero + */ +export const CLEAR_SESSION_BINDING = ` +local binding_key = KEYS[1] +local legacy_provider_key = KEYS[2] +local legacy_owner_key = KEYS[3] +local cooldown_key = KEYS[4] + +local current_key_id = ARGV[1] +local expected_generation = ARGV[2] +local next_generation = ARGV[3] +local expected_provider_id = ARGV[4] +local ttl = tonumber(ARGV[5]) +local cooldown_provider_id = ARGV[6] +local cooldown_ttl = tonumber(ARGV[7]) or 0 + +local function is_positive_integer(value) + local parsed = tonumber(value) + return parsed and parsed > 0 and parsed == math.floor(parsed) +end + +if not ttl or ttl <= 0 or current_key_id == '' or expected_generation == '' or + next_generation == '' or cooldown_ttl < 0 then + return {'conflict', 'invalid_input'} +end +if cooldown_ttl > 0 and + (not is_positive_integer(expected_provider_id) or + cooldown_provider_id ~= expected_provider_id) then + return {'conflict', 'invalid_input'} +end +if redis.call('EXISTS', binding_key) == 0 then + return {'conflict', 'canonical_missing'} +end + +local binding = redis.call('HMGET', binding_key, 'key_id', 'generation', 'provider_id') +local binding_key_id = binding[1] +local generation = binding[2] +local current_provider_id = binding[3] + +if not binding_key_id or not generation or binding_key_id == '' or generation == '' then + return {'conflict', 'canonical_corrupt'} +end +if binding_key_id ~= current_key_id then + return {'conflict', 'canonical_key_mismatch'} +end +if generation ~= expected_generation then + return {'conflict', 'generation_mismatch'} +end +if (current_provider_id or '') ~= expected_provider_id then + return {'conflict', 'provider_mismatch'} +end + +local legacy_owner = redis.call('GET', legacy_owner_key) +local legacy_provider = redis.call('GET', legacy_provider_key) +if current_provider_id and not is_positive_integer(current_provider_id) then + return {'conflict', 'canonical_corrupt'} +end +if legacy_provider and not is_positive_integer(legacy_provider) then + return {'conflict', 'invalid_legacy_provider'} +end +if not legacy_owner then + return {'conflict', 'mirror_missing'} +end +if legacy_owner ~= current_key_id then + return {'conflict', 'foreign_legacy_owner'} +end +if current_provider_id then + if legacy_provider ~= current_provider_id then + return {'conflict', 'mirror_conflict'} + end +elseif legacy_provider then + return {'conflict', 'mirror_conflict'} +end + +redis.call('HSET', binding_key, 'key_id', current_key_id, 'generation', next_generation) +redis.call('HDEL', binding_key, 'provider_id') +redis.call('EXPIRE', binding_key, ttl) +redis.call('SETEX', legacy_owner_key, ttl, current_key_id) +redis.call('DEL', legacy_provider_key) + +if cooldown_ttl > 0 then + redis.call('SETEX', cooldown_key, cooldown_ttl, next_generation) +end + +return {'ok', 'cleared', next_generation, ''} +`; + +/** + * Renew a request-scoped Discovery lease only while the caller still owns it. + * + * KEYS[1]: tenant-scoped Discovery lease key + * ARGV[1]: owner token + * ARGV[2]: lease TTL in seconds + * + * Return: 1 when renewed, otherwise 0. + */ +export const RENEW_SESSION_DISCOVERY_LEASE = ` +local lease_key = KEYS[1] +local owner_token = ARGV[1] +local ttl = tonumber(ARGV[2]) + +if owner_token == '' or not ttl or ttl <= 0 then + return 0 +end +if redis.call('GET', lease_key) ~= owner_token then + return 0 +end + +return redis.call('EXPIRE', lease_key, ttl) +`; + +/** + * Release a request-scoped Discovery lease without deleting a newer owner's + * lease after the original owner's TTL elapsed. + * + * KEYS[1]: tenant-scoped Discovery lease key + * ARGV[1]: owner token + * + * Return: 1 when released, otherwise 0. + */ +export const RELEASE_SESSION_DISCOVERY_LEASE = ` +local lease_key = KEYS[1] +local owner_token = ARGV[1] + +if owner_token == '' or redis.call('GET', lease_key) ~= owner_token then + return 0 +end + +return redis.call('DEL', lease_key) +`; + +/** + * Tenant-authorized administrative termination. Unlike request-level clear, + * this operation intentionally does not compare an old generation. It still + * validates canonical ownership and both legacy mirrors before rotating the + * generation and leaving a null tombstone. + * + * KEYS[1]: canonical binding hash + * KEYS[2]: legacy provider string + * KEYS[3]: legacy key owner string + * ARGV[1]: current key id + * ARGV[2]: next generation + * ARGV[3]: binding TTL in seconds + * ARGV[4]: optional expected provider id for conditional batch termination + */ +export const TERMINATE_SESSION_BINDING = ` +local binding_key = KEYS[1] +local legacy_provider_key = KEYS[2] +local legacy_owner_key = KEYS[3] + +local current_key_id = ARGV[1] +local next_generation = ARGV[2] +local ttl = tonumber(ARGV[3]) +local expected_provider_id = ARGV[4] + +local function is_positive_integer(value) + local parsed = tonumber(value) + return parsed and parsed > 0 and parsed == math.floor(parsed) +end + +if not ttl or ttl <= 0 or current_key_id == '' or next_generation == '' or + (expected_provider_id ~= '' and not is_positive_integer(expected_provider_id)) then + return {'conflict', 'invalid_input'} +end +if redis.call('EXISTS', binding_key) == 0 then + return {'conflict', 'canonical_missing'} +end + +local binding = redis.call('HMGET', binding_key, 'key_id', 'generation', 'provider_id') +local binding_key_id = binding[1] +local generation = binding[2] +local current_provider_id = binding[3] + +if not binding_key_id or not generation or binding_key_id == '' or generation == '' then + return {'conflict', 'canonical_corrupt'} +end +if binding_key_id ~= current_key_id then + return {'conflict', 'canonical_key_mismatch'} +end +if current_provider_id and not is_positive_integer(current_provider_id) then + return {'conflict', 'canonical_corrupt'} +end +if expected_provider_id ~= '' and (current_provider_id or '') ~= expected_provider_id then + return {'conflict', 'provider_mismatch'} +end + +local legacy_owner = redis.call('GET', legacy_owner_key) +local legacy_provider = redis.call('GET', legacy_provider_key) +if legacy_provider and not is_positive_integer(legacy_provider) then + return {'conflict', 'invalid_legacy_provider'} +end +if not legacy_owner then + return {'conflict', 'mirror_missing'} +end +if legacy_owner ~= current_key_id then + return {'conflict', 'foreign_legacy_owner'} +end +if current_provider_id then + if legacy_provider ~= current_provider_id then + return {'conflict', 'mirror_conflict'} + end +elseif legacy_provider then + return {'conflict', 'mirror_conflict'} +end + +redis.call('HSET', binding_key, 'key_id', current_key_id, 'generation', next_generation) +redis.call('HDEL', binding_key, 'provider_id') +redis.call('EXPIRE', binding_key, ttl) +redis.call('SETEX', legacy_owner_key, ttl, current_key_id) +redis.call('DEL', legacy_provider_key) +return {'ok', 'terminated', next_generation, ''} +`; diff --git a/src/lib/redis/session-binding.ts b/src/lib/redis/session-binding.ts new file mode 100644 index 000000000..a72ebb322 --- /dev/null +++ b/src/lib/redis/session-binding.ts @@ -0,0 +1,1539 @@ +import "server-only"; + +import { createHash, randomUUID } from "node:crypto"; +import { logger } from "@/lib/logger"; +import { getRedisClient } from "./client"; +import { + CAS_SESSION_BINDING, + CLEAR_SESSION_BINDING, + DELETE_LEGACY_PROVIDER_IF_VALUE, + READ_OR_RECONCILE_SESSION_BINDING, + RELEASE_SESSION_DISCOVERY_LEASE, + RENEW_SESSION_DISCOVERY_LEASE, + RESTORE_LEGACY_PROVIDER_IF_ABSENT, + TERMINATE_SESSION_BINDING, + TOUCH_SESSION_BINDING, +} from "./lua-scripts"; + +export const DEFAULT_SESSION_BINDING_TTL_SECONDS = 300; +const CAPABILITY_PROBE_TIMEOUT_MS = 5_000; + +export type VersionedBindingCapabilityState = "unknown" | "available" | "unavailable"; + +export type SessionBindingConflictReason = + | "canonical_corrupt" + | "canonical_key_mismatch" + | "canonical_missing" + | "canonical_exists" + | "foreign_legacy_owner" + | "generation_mismatch" + | "invalid_input" + | "invalid_legacy_provider" + | "mirror_conflict" + | "mirror_missing" + | "orphan_legacy_provider" + | "provider_mismatch" + | "unknown_conflict"; + +export type SessionBindingUnavailableReason = + | "capability_probe_failed" + | "capability_unavailable" + | "connection_changed" + | "operation_failed" + | "redis_not_ready"; + +export interface SessionBindingSnapshot { + sessionId: string; + keyId: number; + providerId: number | null; + generation: string; +} + +export interface SessionBindingKeys { + canonical: string; + legacyProvider: string; + legacyOwner: string; +} + +export interface SessionBindingRedisClient { + readonly status: string; + eval(script: string, numberOfKeys: number, ...args: Array): Promise; + evalsha?(sha1: string, numberOfKeys: number, ...args: Array): Promise; + get(key: string): Promise; + hget?(key: string, field: string): Promise; + del(...keys: string[]): Promise; + exists(key: string): Promise; + expire(key: string, ttlSeconds: number): Promise; + on(event: string, listener: (...args: unknown[]) => void): unknown; + off?(event: string, listener: (...args: unknown[]) => void): unknown; + set( + key: string, + value: string, + expiryMode: "EX", + ttlSeconds: number, + condition: "NX" + ): Promise; + setex(key: string, ttlSeconds: number, value: string): Promise; +} + +export interface ReadOrReconcileSessionBindingInput { + sessionId: string; + keyId: number; + ttlSeconds?: number; + redis?: SessionBindingRedisClient; +} + +export interface CompareAndSetSessionBindingInput extends ReadOrReconcileSessionBindingInput { + expectedGeneration: string; + providerId: number; +} + +export interface ClearSessionBindingInput extends ReadOrReconcileSessionBindingInput { + expectedGeneration: string; + expectedProviderId: number | null; + cooldownTtlSeconds?: number; +} + +export interface TouchSessionBindingInput extends ReadOrReconcileSessionBindingInput { + expectedGeneration: string; + expectedProviderId: number | null; +} + +export interface TerminateSessionBindingInput extends ReadOrReconcileSessionBindingInput { + expectedProviderId?: number; +} + +export type LegacySessionBindingMutation = + | { type: "inspect" } + | { type: "refresh" } + | { type: "bind_if_absent"; providerId: number } + | { type: "set"; providerId: number } + | { + type: "clear"; + expectedProviderId?: number | null; + expectedProviderIds?: readonly number[]; + } + | { type: "terminate"; expectedProviderIds?: readonly number[] }; + +export interface LegacySessionBindingMutationInput extends ReadOrReconcileSessionBindingInput { + mutation: LegacySessionBindingMutation; +} + +export type LegacySessionBindingMutationResult = + | { + status: "ok"; + changed: boolean; + providerId: number | null; + /** Provider removed at the scoped terminate linearization point. */ + terminatedProviderId?: number; + } + | SessionBindingConflictResult + | SessionBindingUnavailableResult; + +export interface SessionProviderCooldownInput { + sessionId: string; + keyId: number; + providerId: number; + redis?: SessionBindingRedisClient; +} + +export interface SessionDiscoveryLeaseInput { + sessionId: string; + keyId: number; + ownerToken: string; + redis?: SessionBindingRedisClient; +} + +export interface AcquireSessionDiscoveryLeaseInput { + sessionId: string; + keyId: number; + ttlSeconds: number; + ownerToken?: string; + redis?: SessionBindingRedisClient; +} + +export interface RenewSessionDiscoveryLeaseInput extends SessionDiscoveryLeaseInput { + ttlSeconds: number; +} + +export type SessionDiscoveryLeaseAcquireResult = + | { + status: "acquired"; + ownerToken: string; + legacyFallbackAllowed: false; + } + | { + status: "conflict"; + reason: "invalid_input" | "lease_held"; + legacyFallbackAllowed: false; + } + | SessionBindingUnavailableResult; + +export type SessionDiscoveryLeaseMutationResult = + | { + status: "renewed" | "released"; + legacyFallbackAllowed: false; + } + | { + status: "lost"; + reason: "invalid_input" | "not_owner_or_missing"; + legacyFallbackAllowed: false; + } + | SessionBindingUnavailableResult; + +export interface SessionBindingOkResult { + status: "ok"; + snapshot: SessionBindingSnapshot; + legacyFallbackAllowed: false; + source: + | "created" + | "existing" + | "legacy_upgraded" + | "updated" + | "touched" + | "cleared" + | "terminated"; +} + +export interface SessionBindingConflictResult { + status: "conflict"; + reason: SessionBindingConflictReason; + legacyFallbackAllowed: false; +} + +export interface SessionBindingUnavailableResult { + status: "unavailable"; + reason: SessionBindingUnavailableReason; + capabilityState: VersionedBindingCapabilityState; + legacyFallbackAllowed: boolean; +} + +export type SessionBindingResult = + | SessionBindingOkResult + | SessionBindingConflictResult + | SessionBindingUnavailableResult; +type SessionBindingFailureResult = SessionBindingConflictResult | SessionBindingUnavailableResult; + +export type SessionProviderCooldownResult = + | { + status: "ok"; + coolingDown: boolean; + legacyFallbackAllowed: false; + } + | SessionBindingConflictResult + | SessionBindingUnavailableResult; + +interface CapabilityListeners { + close: (...args: unknown[]) => void; + connect: (...args: unknown[]) => void; + end: (...args: unknown[]) => void; + ready: (...args: unknown[]) => void; + reconnecting: (...args: unknown[]) => void; +} + +interface ReadyClient { + redis: SessionBindingRedisClient; + epoch: number; +} + +const READ_SOURCES = new Set(["created", "existing", "legacy_upgraded"]); +const MUTATION_SOURCES = new Set(["updated", "cleared"]); +const TOUCH_SOURCES = new Set(["touched"]); +const CONFLICT_REASONS = new Set([ + "canonical_corrupt", + "canonical_exists", + "canonical_key_mismatch", + "canonical_missing", + "foreign_legacy_owner", + "generation_mismatch", + "invalid_input", + "invalid_legacy_provider", + "mirror_conflict", + "mirror_missing", + "orphan_legacy_provider", + "provider_mismatch", +]); + +let capabilityClient: SessionBindingRedisClient | null = null; +let capabilityState: VersionedBindingCapabilityState = "unknown"; +let capabilityEpoch = 0; +let capabilityProbe: Promise | null = null; +let capabilityListeners: CapabilityListeners | null = null; +const scriptSha1Cache = new Map(); + +function namespacedKey(namespace: string | undefined, key: string): string { + return namespace ? `${namespace}:${key}` : key; +} + +function bindingHashTag(sessionId: string, keyId: number): string { + return createHash("sha256").update(`${keyId}\0${sessionId}`).digest("hex"); +} + +export function buildCanonicalSessionBindingKey( + sessionId: string, + keyId: number, + namespace?: string +): string { + return namespacedKey( + namespace, + `session-binding:v1:{${bindingHashTag(sessionId, keyId)}}:binding` + ); +} + +export function buildLegacySessionProviderKey(sessionId: string, namespace?: string): string { + return namespacedKey(namespace, `session:${sessionId}:provider`); +} + +export function buildLegacySessionOwnerKey(sessionId: string, namespace?: string): string { + return namespacedKey(namespace, `session:${sessionId}:key`); +} + +export function buildSessionProviderCooldownKey( + sessionId: string, + keyId: number, + providerId: number, + namespace?: string +): string { + return namespacedKey( + namespace, + `session-binding:v1:{${bindingHashTag(sessionId, keyId)}}:provider:${providerId}:cooldown` + ); +} + +export function buildSessionDiscoveryLeaseKey( + sessionId: string, + keyId: number, + namespace?: string +): string { + return namespacedKey( + namespace, + `session-binding:v1:{${bindingHashTag(sessionId, keyId)}}:discovery-lease` + ); +} + +export function buildSessionBindingKeys( + sessionId: string, + keyId: number, + namespace?: string +): SessionBindingKeys { + return { + canonical: buildCanonicalSessionBindingKey(sessionId, keyId, namespace), + legacyProvider: buildLegacySessionProviderKey(sessionId, namespace), + legacyOwner: buildLegacySessionOwnerKey(sessionId, namespace), + }; +} + +function isPositiveInteger(value: number): boolean { + return Number.isSafeInteger(value) && value > 0; +} + +function isValidIdentity(sessionId: string, keyId: number, ttlSeconds: number): boolean { + return sessionId.length > 0 && isPositiveInteger(keyId) && isPositiveInteger(ttlSeconds); +} + +function conflict(reason: SessionBindingConflictReason): SessionBindingConflictResult { + return { status: "conflict", reason, legacyFallbackAllowed: false }; +} + +function unavailable(reason: SessionBindingUnavailableReason): SessionBindingUnavailableResult { + return { + status: "unavailable", + reason, + capabilityState, + legacyFallbackAllowed: + reason === "capability_probe_failed" || + reason === "capability_unavailable" || + reason === "redis_not_ready", + }; +} + +function normalizeEvalResult(raw: unknown): string[] { + if (!Array.isArray(raw) || raw.length < 2) { + throw new Error("Invalid session binding Lua result"); + } + return raw.map((value) => { + if (typeof value === "string" || typeof value === "number") { + return String(value); + } + if (Buffer.isBuffer(value)) { + return value.toString("utf8"); + } + throw new Error("Invalid value in session binding Lua result"); + }); +} + +function parseLeaseMutationFlag(raw: unknown): boolean { + const value = Buffer.isBuffer(raw) ? raw.toString("utf8") : raw; + if (value === 1 || value === "1") return true; + if (value === 0 || value === "0" || value === null) return false; + throw new Error("Invalid session Discovery lease Lua result"); +} + +function parseProviderId(raw: string): number | null { + if (raw === "") return null; + const providerId = Number(raw); + if (!isPositiveInteger(providerId)) { + throw new Error("Invalid provider id in session binding Lua result"); + } + return providerId; +} + +function parseBindingResult( + raw: unknown, + identity: { sessionId: string; keyId: number }, + allowedSources: Set +): SessionBindingResult { + const values = normalizeEvalResult(raw); + if (values[0] === "conflict") { + const reason = CONFLICT_REASONS.has(values[1] as SessionBindingConflictReason) + ? (values[1] as SessionBindingConflictReason) + : "unknown_conflict"; + return conflict(reason); + } + if (values[0] !== "ok" || values.length < 4 || !allowedSources.has(values[1])) { + throw new Error("Unexpected session binding Lua result"); + } + + const generation = values[2]; + if (!generation) { + throw new Error("Missing generation in session binding Lua result"); + } + + return { + status: "ok", + source: values[1] as SessionBindingOkResult["source"], + snapshot: { + ...identity, + generation, + providerId: parseProviderId(values[3]), + }, + legacyFallbackAllowed: false, + }; +} + +function detachCapabilityListeners(): void { + if (!capabilityClient || !capabilityListeners || !capabilityClient.off) return; + capabilityClient.off("close", capabilityListeners.close); + capabilityClient.off("connect", capabilityListeners.connect); + capabilityClient.off("end", capabilityListeners.end); + capabilityClient.off("ready", capabilityListeners.ready); + capabilityClient.off("reconnecting", capabilityListeners.reconnecting); +} + +function resetCapabilityForConnection(client: SessionBindingRedisClient): void { + if (capabilityClient !== client) return; + capabilityEpoch += 1; + capabilityState = "unknown"; + capabilityProbe = null; +} + +function attachCapabilityClient(client: SessionBindingRedisClient): void { + if (capabilityClient === client) return; + + detachCapabilityListeners(); + capabilityClient = client; + capabilityEpoch += 1; + capabilityState = "unknown"; + capabilityProbe = null; + + const listeners: CapabilityListeners = { + close: () => resetCapabilityForConnection(client), + connect: () => resetCapabilityForConnection(client), + end: () => resetCapabilityForConnection(client), + ready: () => { + resetCapabilityForConnection(client); + void ensureVersionedBindingCapability(client); + }, + reconnecting: () => resetCapabilityForConnection(client), + }; + capabilityListeners = listeners; + client.on("close", listeners.close); + client.on("connect", listeners.connect); + client.on("end", listeners.end); + client.on("ready", listeners.ready); + client.on("reconnecting", listeners.reconnecting); +} + +function currentRedisClient( + override?: SessionBindingRedisClient +): SessionBindingRedisClient | null { + if (override) return override; + return getRedisClient({ allowWhenRateLimitDisabled: true }) as SessionBindingRedisClient | null; +} + +function scriptSha1(script: string): string { + const cached = scriptSha1Cache.get(script); + if (cached) return cached; + const digest = createHash("sha1").update(script).digest("hex"); + scriptSha1Cache.set(script, digest); + return digest; +} + +async function evalBindingScript( + redis: SessionBindingRedisClient, + script: string, + numberOfKeys: number, + ...args: Array +): Promise { + if (!redis.evalsha) return redis.eval(script, numberOfKeys, ...args); + try { + return await redis.evalsha(scriptSha1(script), numberOfKeys, ...args); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + if (!/NOSCRIPT/i.test(message)) throw error; + return redis.eval(script, numberOfKeys, ...args); + } +} + +function markCapabilityUnavailable( + client: SessionBindingRedisClient, + epoch: number, + error: unknown +): void { + if (capabilityClient !== client || capabilityEpoch !== epoch) return; + capabilityState = "unavailable"; + capabilityProbe = null; + logger.warn("Versioned session binding Redis capability is unavailable", { + error: error instanceof Error ? error.message : String(error), + }); +} + +function isCapabilityError(error: unknown): boolean { + const message = error instanceof Error ? error.message : String(error); + return /CROSSSLOT|NOPERM|unknown command|EVAL.*(?:disabled|not allowed)|script execution disabled|Lua scripts? (?:are )?disabled/i.test( + message + ); +} + +function isBindingDataError(error: unknown): boolean { + const message = error instanceof Error ? error.message : String(error); + return ( + message.startsWith("Invalid session binding Lua result") || + message.startsWith("Invalid value in session binding Lua result") || + message.startsWith("Unexpected session binding Lua result") || + message.startsWith("Missing generation in session binding Lua result") || + message.startsWith("Invalid provider id in session binding Lua result") || + message.includes("WRONGTYPE") + ); +} + +function handleOperationError(ready: ReadyClient, error: unknown): SessionBindingFailureResult { + if (isCapabilityError(error)) { + markCapabilityUnavailable(ready.redis, ready.epoch, error); + return unavailable("capability_unavailable"); + } + + if (isBindingDataError(error)) { + logger.warn("Versioned session binding data is invalid", { + error: error instanceof Error ? error.message : String(error), + }); + return conflict("canonical_corrupt"); + } + + logger.warn("Versioned session binding operation failed", { + error: error instanceof Error ? error.message : String(error), + }); + return unavailable("operation_failed"); +} + +function handleLeaseOperationError( + ready: ReadyClient, + error: unknown +): SessionBindingUnavailableResult { + if (isCapabilityError(error)) { + markCapabilityUnavailable(ready.redis, ready.epoch, error); + return unavailable("capability_unavailable"); + } + + logger.warn("Session Discovery lease operation failed", { + error: error instanceof Error ? error.message : String(error), + }); + return unavailable("operation_failed"); +} + +async function runCapabilityProbe( + client: SessionBindingRedisClient, + epoch: number +): Promise { + const deadlineAt = Date.now() + CAPABILITY_PROBE_TIMEOUT_MS; + const namespace = `session-binding-capability-probe:${randomUUID()}`; + const sessionId = "probe-session"; + const keyId = 1; + const providerId = 1; + const ttlSeconds = 60; + const keys = buildSessionBindingKeys(sessionId, keyId, namespace); + const cooldownKey = buildSessionProviderCooldownKey(sessionId, keyId, providerId, namespace); + const leaseKey = buildSessionDiscoveryLeaseKey(sessionId, keyId, namespace); + const cleanupKeys = [ + keys.canonical, + keys.legacyProvider, + keys.legacyOwner, + cooldownKey, + leaseKey, + ]; + const initialGeneration = randomUUID(); + const boundGeneration = randomUUID(); + const clearedGeneration = randomUUID(); + const leaseOwnerToken = randomUUID(); + let operationsSucceeded = false; + let cleanupSucceeded = false; + + try { + const read = parseBindingResult( + await withCapabilityProbeDeadline( + client.eval( + READ_OR_RECONCILE_SESSION_BINDING, + 3, + keys.canonical, + keys.legacyProvider, + keys.legacyOwner, + keyId.toString(), + initialGeneration, + ttlSeconds.toString() + ), + deadlineAt + ), + { sessionId, keyId }, + READ_SOURCES + ); + if (read.status !== "ok" || read.source !== "created") { + throw new Error("Session binding capability probe reconcile failed"); + } + + const updated = parseBindingResult( + await withCapabilityProbeDeadline( + client.eval( + CAS_SESSION_BINDING, + 3, + keys.canonical, + keys.legacyProvider, + keys.legacyOwner, + keyId.toString(), + read.snapshot.generation, + boundGeneration, + providerId.toString(), + ttlSeconds.toString() + ), + deadlineAt + ), + { sessionId, keyId }, + MUTATION_SOURCES + ); + if (updated.status !== "ok" || updated.source !== "updated") { + throw new Error("Session binding capability probe update failed"); + } + + const touched = parseBindingResult( + await withCapabilityProbeDeadline( + client.eval( + TOUCH_SESSION_BINDING, + 3, + keys.canonical, + keys.legacyProvider, + keys.legacyOwner, + keyId.toString(), + updated.snapshot.generation, + providerId.toString(), + ttlSeconds.toString() + ), + deadlineAt + ), + { sessionId, keyId }, + TOUCH_SOURCES + ); + if ( + touched.status !== "ok" || + touched.source !== "touched" || + touched.snapshot.generation !== updated.snapshot.generation || + touched.snapshot.providerId !== providerId + ) { + throw new Error("Session binding capability probe touch failed"); + } + + const cleared = parseBindingResult( + await withCapabilityProbeDeadline( + client.eval( + CLEAR_SESSION_BINDING, + 4, + keys.canonical, + keys.legacyProvider, + keys.legacyOwner, + cooldownKey, + keyId.toString(), + updated.snapshot.generation, + clearedGeneration, + providerId.toString(), + ttlSeconds.toString(), + providerId.toString(), + ttlSeconds.toString() + ), + deadlineAt + ), + { sessionId, keyId }, + MUTATION_SOURCES + ); + if (cleared.status !== "ok" || cleared.source !== "cleared") { + throw new Error("Session binding capability probe clear failed"); + } + + const cooldownGeneration = await withCapabilityProbeDeadline( + client.get(cooldownKey), + deadlineAt + ); + if (cooldownGeneration !== cleared.snapshot.generation) { + throw new Error("Session binding capability probe cooldown failed"); + } + + const acquiredLease = await withCapabilityProbeDeadline( + client.set(leaseKey, leaseOwnerToken, "EX", ttlSeconds, "NX"), + deadlineAt + ); + if (acquiredLease !== "OK") { + throw new Error("Session binding capability probe lease acquire failed"); + } + + const renewedLease = parseLeaseMutationFlag( + await withCapabilityProbeDeadline( + client.eval( + RENEW_SESSION_DISCOVERY_LEASE, + 1, + leaseKey, + leaseOwnerToken, + ttlSeconds.toString() + ), + deadlineAt + ) + ); + if (!renewedLease) { + throw new Error("Session binding capability probe lease renew failed"); + } + + const releasedLease = parseLeaseMutationFlag( + await withCapabilityProbeDeadline( + client.eval(RELEASE_SESSION_DISCOVERY_LEASE, 1, leaseKey, leaseOwnerToken), + deadlineAt + ) + ); + if (!releasedLease) { + throw new Error("Session binding capability probe lease release failed"); + } + operationsSucceeded = capabilityClient === client && capabilityEpoch === epoch; + } catch (error) { + logger.warn("Versioned session binding Redis capability probe failed", { + error: error instanceof Error ? error.message : String(error), + }); + } finally { + try { + await withCapabilityProbeDeadline( + Promise.all(cleanupKeys.map((key) => client.del(key))), + deadlineAt + ); + cleanupSucceeded = true; + } catch (error) { + logger.warn("Versioned session binding Redis capability probe cleanup failed", { + error: error instanceof Error ? error.message : String(error), + }); + } + } + + return operationsSucceeded && cleanupSucceeded; +} + +async function withCapabilityProbeDeadline( + operation: Promise, + deadlineAt: number +): Promise { + // The timeout races the Redis command but cannot cancel it. Observe a late + // rejection so an operation that settles after the deadline never becomes + // an unhandled promise rejection. + operation.catch(() => {}); + const remainingMs = deadlineAt - Date.now(); + if (remainingMs <= 0) throw new Error("Session binding capability probe deadline exceeded"); + + let timeout: ReturnType | undefined; + const deadline = new Promise((_, reject) => { + timeout = setTimeout( + () => reject(new Error("Session binding capability probe deadline exceeded")), + remainingMs + ); + timeout.unref?.(); + }); + try { + return await Promise.race([operation, deadline]); + } finally { + if (timeout) clearTimeout(timeout); + } +} + +export function getVersionedBindingCapabilityState(): VersionedBindingCapabilityState { + return capabilityState; +} + +export async function ensureVersionedBindingCapability( + redisOverride?: SessionBindingRedisClient +): Promise { + const redis = currentRedisClient(redisOverride); + if (!redis) return "unknown"; + + attachCapabilityClient(redis); + if (redis.status !== "ready") return capabilityState; + if (capabilityState !== "unknown") return capabilityState; + if (capabilityProbe) return capabilityProbe; + + const epoch = capabilityEpoch; + const probe = (async () => { + const supported = await runCapabilityProbe(redis, epoch); + if (capabilityClient !== redis || capabilityEpoch !== epoch) { + return capabilityState; + } + capabilityState = supported ? "available" : "unavailable"; + return capabilityState; + })(); + capabilityProbe = probe; + + try { + return await probe; + } finally { + if (capabilityProbe === probe) capabilityProbe = null; + } +} + +async function readyVersionedClient( + redisOverride?: SessionBindingRedisClient +): Promise { + const redis = currentRedisClient(redisOverride); + if (!redis) return unavailable("redis_not_ready"); + + attachCapabilityClient(redis); + if (redis.status !== "ready") return unavailable("redis_not_ready"); + + const state = await ensureVersionedBindingCapability(redis); + if (state !== "available") { + return unavailable( + state === "unavailable" ? "capability_unavailable" : "capability_probe_failed" + ); + } + return { redis, epoch: capabilityEpoch }; +} + +function connectionIsCurrent(ready: ReadyClient): boolean { + return capabilityClient === ready.redis && capabilityEpoch === ready.epoch; +} + +export async function acquireSessionDiscoveryLease( + input: AcquireSessionDiscoveryLeaseInput +): Promise { + const ownerToken = input.ownerToken ?? randomUUID(); + if (!isValidIdentity(input.sessionId, input.keyId, input.ttlSeconds) || ownerToken.length === 0) { + return { + status: "conflict", + reason: "invalid_input", + legacyFallbackAllowed: false, + }; + } + + const ready = await readyVersionedClient(input.redis); + if ("status" in ready) return ready; + + try { + const result = await ready.redis.set( + buildSessionDiscoveryLeaseKey(input.sessionId, input.keyId), + ownerToken, + "EX", + input.ttlSeconds, + "NX" + ); + if (!connectionIsCurrent(ready)) return unavailable("connection_changed"); + if (result === "OK" || (Buffer.isBuffer(result) && result.toString("utf8") === "OK")) { + return { status: "acquired", ownerToken, legacyFallbackAllowed: false }; + } + if (result === null) { + return { status: "conflict", reason: "lease_held", legacyFallbackAllowed: false }; + } + return handleLeaseOperationError(ready, new Error("Unexpected Discovery lease SET result")); + } catch (error) { + return handleLeaseOperationError(ready, error); + } +} + +export async function renewSessionDiscoveryLease( + input: RenewSessionDiscoveryLeaseInput +): Promise { + if ( + !isValidIdentity(input.sessionId, input.keyId, input.ttlSeconds) || + input.ownerToken.length === 0 + ) { + return { status: "lost", reason: "invalid_input", legacyFallbackAllowed: false }; + } + + const ready = await readyVersionedClient(input.redis); + if ("status" in ready) return ready; + + try { + const renewed = parseLeaseMutationFlag( + await evalBindingScript( + ready.redis, + RENEW_SESSION_DISCOVERY_LEASE, + 1, + buildSessionDiscoveryLeaseKey(input.sessionId, input.keyId), + input.ownerToken, + input.ttlSeconds.toString() + ) + ); + if (!connectionIsCurrent(ready)) return unavailable("connection_changed"); + return renewed + ? { status: "renewed", legacyFallbackAllowed: false } + : { + status: "lost", + reason: "not_owner_or_missing", + legacyFallbackAllowed: false, + }; + } catch (error) { + return handleLeaseOperationError(ready, error); + } +} + +export async function releaseSessionDiscoveryLease( + input: SessionDiscoveryLeaseInput +): Promise { + if ( + input.sessionId.length === 0 || + !isPositiveInteger(input.keyId) || + input.ownerToken.length === 0 + ) { + return { status: "lost", reason: "invalid_input", legacyFallbackAllowed: false }; + } + + const ready = await readyVersionedClient(input.redis); + if ("status" in ready) return ready; + + try { + const released = parseLeaseMutationFlag( + await evalBindingScript( + ready.redis, + RELEASE_SESSION_DISCOVERY_LEASE, + 1, + buildSessionDiscoveryLeaseKey(input.sessionId, input.keyId), + input.ownerToken + ) + ); + if (!connectionIsCurrent(ready)) return unavailable("connection_changed"); + return released + ? { status: "released", legacyFallbackAllowed: false } + : { + status: "lost", + reason: "not_owner_or_missing", + legacyFallbackAllowed: false, + }; + } catch (error) { + return handleLeaseOperationError(ready, error); + } +} + +export async function readOrReconcileSessionBinding( + input: ReadOrReconcileSessionBindingInput +): Promise { + const ttlSeconds = input.ttlSeconds ?? DEFAULT_SESSION_BINDING_TTL_SECONDS; + if (!isValidIdentity(input.sessionId, input.keyId, ttlSeconds)) { + return conflict("invalid_input"); + } + + const ready = await readyVersionedClient(input.redis); + if ("status" in ready) return ready; + + const keys = buildSessionBindingKeys(input.sessionId, input.keyId); + try { + const raw = await evalBindingScript( + ready.redis, + READ_OR_RECONCILE_SESSION_BINDING, + 3, + keys.canonical, + keys.legacyProvider, + keys.legacyOwner, + input.keyId.toString(), + randomUUID(), + ttlSeconds.toString() + ); + if (!connectionIsCurrent(ready)) return unavailable("connection_changed"); + return parseBindingResult( + raw, + { sessionId: input.sessionId, keyId: input.keyId }, + READ_SOURCES + ); + } catch (error) { + return handleOperationError(ready, error); + } +} + +export async function refreshSessionBinding( + input: ReadOrReconcileSessionBindingInput +): Promise { + return readOrReconcileSessionBinding(input); +} + +export async function compareAndSetSessionBinding( + input: CompareAndSetSessionBindingInput +): Promise { + const ttlSeconds = input.ttlSeconds ?? DEFAULT_SESSION_BINDING_TTL_SECONDS; + if ( + !isValidIdentity(input.sessionId, input.keyId, ttlSeconds) || + !input.expectedGeneration || + !isPositiveInteger(input.providerId) + ) { + return conflict("invalid_input"); + } + + const ready = await readyVersionedClient(input.redis); + if ("status" in ready) return ready; + + const keys = buildSessionBindingKeys(input.sessionId, input.keyId); + try { + const raw = await evalBindingScript( + ready.redis, + CAS_SESSION_BINDING, + 3, + keys.canonical, + keys.legacyProvider, + keys.legacyOwner, + input.keyId.toString(), + input.expectedGeneration, + randomUUID(), + input.providerId.toString(), + ttlSeconds.toString() + ); + if (!connectionIsCurrent(ready)) return unavailable("connection_changed"); + return parseBindingResult( + raw, + { sessionId: input.sessionId, keyId: input.keyId }, + MUTATION_SOURCES + ); + } catch (error) { + return handleOperationError(ready, error); + } +} + +export async function touchSessionBinding( + input: TouchSessionBindingInput +): Promise { + const ttlSeconds = input.ttlSeconds ?? DEFAULT_SESSION_BINDING_TTL_SECONDS; + if ( + !isValidIdentity(input.sessionId, input.keyId, ttlSeconds) || + !input.expectedGeneration || + (input.expectedProviderId !== null && !isPositiveInteger(input.expectedProviderId)) + ) { + return conflict("invalid_input"); + } + + const ready = await readyVersionedClient(input.redis); + if ("status" in ready) return ready; + + const keys = buildSessionBindingKeys(input.sessionId, input.keyId); + try { + const raw = await evalBindingScript( + ready.redis, + TOUCH_SESSION_BINDING, + 3, + keys.canonical, + keys.legacyProvider, + keys.legacyOwner, + input.keyId.toString(), + input.expectedGeneration, + input.expectedProviderId?.toString() ?? "", + ttlSeconds.toString() + ); + if (!connectionIsCurrent(ready)) return unavailable("connection_changed"); + return parseBindingResult( + raw, + { sessionId: input.sessionId, keyId: input.keyId }, + TOUCH_SOURCES + ); + } catch (error) { + return handleOperationError(ready, error); + } +} + +export async function clearSessionBinding( + input: ClearSessionBindingInput +): Promise { + const ttlSeconds = input.ttlSeconds ?? DEFAULT_SESSION_BINDING_TTL_SECONDS; + const cooldownTtlSeconds = input.cooldownTtlSeconds ?? 0; + if ( + !isValidIdentity(input.sessionId, input.keyId, ttlSeconds) || + !input.expectedGeneration || + (input.expectedProviderId !== null && !isPositiveInteger(input.expectedProviderId)) || + !Number.isSafeInteger(cooldownTtlSeconds) || + cooldownTtlSeconds < 0 || + (cooldownTtlSeconds > 0 && input.expectedProviderId === null) + ) { + return conflict("invalid_input"); + } + + const ready = await readyVersionedClient(input.redis); + if ("status" in ready) return ready; + + const keys = buildSessionBindingKeys(input.sessionId, input.keyId); + const cooldownKey = + input.expectedProviderId === null + ? keys.canonical + : buildSessionProviderCooldownKey(input.sessionId, input.keyId, input.expectedProviderId); + const expectedProviderId = input.expectedProviderId?.toString() ?? ""; + + try { + const raw = await evalBindingScript( + ready.redis, + CLEAR_SESSION_BINDING, + 4, + keys.canonical, + keys.legacyProvider, + keys.legacyOwner, + cooldownKey, + input.keyId.toString(), + input.expectedGeneration, + randomUUID(), + expectedProviderId, + ttlSeconds.toString(), + cooldownTtlSeconds > 0 ? expectedProviderId : "", + cooldownTtlSeconds.toString() + ); + if (!connectionIsCurrent(ready)) return unavailable("connection_changed"); + return parseBindingResult( + raw, + { sessionId: input.sessionId, keyId: input.keyId }, + MUTATION_SOURCES + ); + } catch (error) { + return handleOperationError(ready, error); + } +} + +export async function terminateSessionBinding( + input: TerminateSessionBindingInput +): Promise { + const ttlSeconds = input.ttlSeconds ?? DEFAULT_SESSION_BINDING_TTL_SECONDS; + if ( + !isValidIdentity(input.sessionId, input.keyId, ttlSeconds) || + (input.expectedProviderId !== undefined && !isPositiveInteger(input.expectedProviderId)) + ) { + return conflict("invalid_input"); + } + + const ready = await readyVersionedClient(input.redis); + if ("status" in ready) return ready; + + const keys = buildSessionBindingKeys(input.sessionId, input.keyId); + try { + const raw = await evalBindingScript( + ready.redis, + TERMINATE_SESSION_BINDING, + 3, + keys.canonical, + keys.legacyProvider, + keys.legacyOwner, + input.keyId.toString(), + randomUUID(), + ttlSeconds.toString(), + input.expectedProviderId?.toString() ?? "" + ); + if (!connectionIsCurrent(ready)) return unavailable("connection_changed"); + return parseBindingResult( + raw, + { sessionId: input.sessionId, keyId: input.keyId }, + new Set(["terminated"]) + ); + } catch (error) { + return handleOperationError(ready, error); + } +} + +function parseLegacyProviderId(raw: string | null): number | null | undefined { + if (raw === null) return null; + const providerId = Number(raw); + return isPositiveInteger(providerId) ? providerId : undefined; +} + +async function deleteLegacyProviderIfValue( + redis: SessionBindingRedisClient, + providerKey: string, + providerId: number +): Promise { + const result = await redis.eval( + DELETE_LEGACY_PROVIDER_IF_VALUE, + 1, + providerKey, + providerId.toString() + ); + const normalized = Buffer.isBuffer(result) ? result.toString("utf8") : result; + if (normalized === 1 || normalized === "1") return true; + if (normalized === 0 || normalized === "0" || normalized === null) return false; + throw new Error("Invalid conditional legacy provider delete result"); +} + +async function ensureLegacyOwner( + redis: SessionBindingRedisClient, + keys: SessionBindingKeys, + keyId: number, + ttlSeconds: number +): Promise { + if ((await redis.expire(keys.legacyOwner, ttlSeconds)) > 0) return null; + + await redis.set(keys.legacyOwner, keyId.toString(), "EX", ttlSeconds, "NX"); + const owner = await redis.get(keys.legacyOwner); + if (owner === keyId.toString()) return null; + return conflict(owner === null ? "orphan_legacy_provider" : "foreign_legacy_owner"); +} + +/** + * A legacy fallback mutation can race a different worker which has already + * recovered versioned binding capability. Re-check the canonical key after a + * legacy write and fail closed if the versioned owner appeared in between. + * Rollback uses a single-key conditional Lua script, so a concurrent versioned + * writer cannot have its newer provider value deleted. If scripts are disabled + * entirely, the mutation still fails closed and leaves the mirror untouched. + */ +async function rejectLegacyMutationAfterCanonicalAppeared( + redis: SessionBindingRedisClient, + keys: SessionBindingKeys, + rollbackProviderValue?: string, + restoreProviderValue?: { value: string; ttlSeconds: number } +): Promise { + if ((await redis.exists(keys.canonical)) === 0) return null; + + if (rollbackProviderValue !== undefined) { + try { + // A recovered versioned worker may have imported this exact provider + // into canonical before the post-write check. In that case the legacy + // mirror is already part of valid dual-write state and must survive the + // rollback; deleting it would manufacture mirror_missing. + const canonicalProvider = await redis.hget?.(keys.canonical, "provider_id"); + if (canonicalProvider !== rollbackProviderValue && canonicalProvider !== undefined) { + await redis.eval( + DELETE_LEGACY_PROVIDER_IF_VALUE, + 1, + keys.legacyProvider, + rollbackProviderValue + ); + } + } catch (error) { + logger.warn("Legacy binding rollback could not execute atomically", { + error: error instanceof Error ? error.message : String(error), + }); + } + } + + if (restoreProviderValue !== undefined) { + try { + // The canonical hash and legacy mirror intentionally use different + // cluster slots. Read the canonical provider first, then use a + // single-key conditional write so we only restore a mirror that still + // belongs to the canonical binding observed by this fallback path. + const canonicalProvider = await redis.hget?.(keys.canonical, "provider_id"); + if (canonicalProvider === restoreProviderValue.value) { + await redis.eval( + RESTORE_LEGACY_PROVIDER_IF_ABSENT, + 1, + keys.legacyProvider, + restoreProviderValue.value, + restoreProviderValue.ttlSeconds.toString() + ); + } + } catch (error) { + logger.warn("Legacy binding mirror restoration could not execute atomically", { + error: error instanceof Error ? error.message : String(error), + }); + } + } + + return conflict("canonical_exists"); +} + +export async function mutateLegacySessionBindingSafely( + input: LegacySessionBindingMutationInput +): Promise { + const ttlSeconds = input.ttlSeconds ?? DEFAULT_SESSION_BINDING_TTL_SECONDS; + if (!isValidIdentity(input.sessionId, input.keyId, ttlSeconds)) { + return conflict("invalid_input"); + } + + const providerFromMutation = + input.mutation.type === "bind_if_absent" || input.mutation.type === "set" + ? input.mutation.providerId + : null; + if (providerFromMutation !== null && !isPositiveInteger(providerFromMutation)) { + return conflict("invalid_input"); + } + if ( + input.mutation.type === "terminate" && + input.mutation.expectedProviderIds?.some((providerId) => !isPositiveInteger(providerId)) + ) { + return conflict("invalid_input"); + } + if ( + input.mutation.type === "clear" && + ((input.mutation.expectedProviderId != null && + !isPositiveInteger(input.mutation.expectedProviderId)) || + input.mutation.expectedProviderIds?.some((providerId) => !isPositiveInteger(providerId)) || + (input.mutation.expectedProviderId != null && + input.mutation.expectedProviderIds !== undefined)) + ) { + return conflict("invalid_input"); + } + + const redis = currentRedisClient(input.redis); + if (!redis || redis.status !== "ready") return unavailable("redis_not_ready"); + + const keys = buildSessionBindingKeys(input.sessionId, input.keyId); + try { + if ((await redis.exists(keys.canonical)) > 0) { + return conflict("canonical_exists"); + } + + let [legacyOwner, legacyProviderRaw] = await Promise.all([ + redis.get(keys.legacyOwner), + redis.get(keys.legacyProvider), + ]); + if (legacyOwner === null) { + if (legacyProviderRaw !== null) return conflict("orphan_legacy_provider"); + await redis.set(keys.legacyOwner, input.keyId.toString(), "EX", ttlSeconds, "NX"); + [legacyOwner, legacyProviderRaw] = await Promise.all([ + redis.get(keys.legacyOwner), + redis.get(keys.legacyProvider), + ]); + } + + if (legacyOwner !== input.keyId.toString()) return conflict("foreign_legacy_owner"); + const legacyProvider = parseLegacyProviderId(legacyProviderRaw); + if (legacyProvider === undefined) return conflict("invalid_legacy_provider"); + if ((await redis.exists(keys.canonical)) > 0) return conflict("canonical_exists"); + + const ownerRefreshConflict = await ensureLegacyOwner(redis, keys, input.keyId, ttlSeconds); + if (ownerRefreshConflict) return ownerRefreshConflict; + + switch (input.mutation.type) { + case "inspect": + return { status: "ok", changed: false, providerId: legacyProvider }; + case "refresh": + await redis.expire(keys.legacyOwner, ttlSeconds); + if (legacyProvider !== null) await redis.expire(keys.legacyProvider, ttlSeconds); + { + const conflictAfterRefresh = await rejectLegacyMutationAfterCanonicalAppeared( + redis, + keys + ); + if (conflictAfterRefresh) return conflictAfterRefresh; + } + return { status: "ok", changed: false, providerId: legacyProvider }; + case "bind_if_absent": { + if (legacyProvider !== null) { + return { status: "ok", changed: false, providerId: legacyProvider }; + } + const result = await redis.set( + keys.legacyProvider, + input.mutation.providerId.toString(), + "EX", + ttlSeconds, + "NX" + ); + if (result === "OK") { + const ownerConflict = await ensureLegacyOwner(redis, keys, input.keyId, ttlSeconds); + if (ownerConflict) { + await redis.eval( + DELETE_LEGACY_PROVIDER_IF_VALUE, + 1, + keys.legacyProvider, + input.mutation.providerId.toString() + ); + return ownerConflict; + } + const conflictAfterBind = await rejectLegacyMutationAfterCanonicalAppeared( + redis, + keys, + input.mutation.providerId.toString() + ); + if (conflictAfterBind) return conflictAfterBind; + return { status: "ok", changed: true, providerId: input.mutation.providerId }; + } + const concurrentProvider = parseLegacyProviderId(await redis.get(keys.legacyProvider)); + if (concurrentProvider === undefined) return conflict("invalid_legacy_provider"); + return { status: "ok", changed: false, providerId: concurrentProvider }; + } + case "set": + await redis.setex(keys.legacyProvider, ttlSeconds, input.mutation.providerId.toString()); + { + const ownerConflict = await ensureLegacyOwner(redis, keys, input.keyId, ttlSeconds); + if (ownerConflict) { + await redis.eval( + DELETE_LEGACY_PROVIDER_IF_VALUE, + 1, + keys.legacyProvider, + input.mutation.providerId.toString() + ); + return ownerConflict; + } + const conflictAfterSet = await rejectLegacyMutationAfterCanonicalAppeared( + redis, + keys, + input.mutation.providerId.toString() + ); + if (conflictAfterSet) return conflictAfterSet; + } + return { status: "ok", changed: true, providerId: input.mutation.providerId }; + case "clear": + if ( + (input.mutation.expectedProviderId != null && + legacyProvider !== input.mutation.expectedProviderId) || + (input.mutation.expectedProviderIds && + (legacyProvider === null || + !input.mutation.expectedProviderIds.includes(legacyProvider))) + ) { + return conflict("provider_mismatch"); + } + if (legacyProvider === null) { + return { status: "ok", changed: false, providerId: null }; + } + { + const deleted = await deleteLegacyProviderIfValue( + redis, + keys.legacyProvider, + legacyProvider + ); + if (!deleted) { + const conflictAfterConcurrentMutation = + await rejectLegacyMutationAfterCanonicalAppeared(redis, keys); + if (conflictAfterConcurrentMutation) return conflictAfterConcurrentMutation; + + const concurrentProvider = parseLegacyProviderId(await redis.get(keys.legacyProvider)); + if (concurrentProvider === undefined) return conflict("invalid_legacy_provider"); + if (concurrentProvider === null) { + await redis.expire(keys.legacyOwner, ttlSeconds); + return { status: "ok", changed: false, providerId: null }; + } + return conflict("provider_mismatch"); + } + } + await redis.expire(keys.legacyOwner, ttlSeconds); + { + const conflictAfterClear = await rejectLegacyMutationAfterCanonicalAppeared( + redis, + keys, + undefined, + { value: legacyProvider.toString(), ttlSeconds } + ); + if (conflictAfterClear) return conflictAfterClear; + } + return { status: "ok", changed: true, providerId: null }; + case "terminate": + if ( + input.mutation.expectedProviderIds && + (legacyProvider === null || !input.mutation.expectedProviderIds.includes(legacyProvider)) + ) { + return conflict("provider_mismatch"); + } + if (input.mutation.expectedProviderIds) { + const providerToTerminate = legacyProvider as number; + const deleted = await deleteLegacyProviderIfValue( + redis, + keys.legacyProvider, + providerToTerminate + ); + if (!deleted) { + const conflictAfterConcurrentMutation = + await rejectLegacyMutationAfterCanonicalAppeared(redis, keys); + if (conflictAfterConcurrentMutation) return conflictAfterConcurrentMutation; + return conflict("provider_mismatch"); + } + + // Provider-scoped invalidation must preserve tenant ownership. A + // concurrent failover may install a new Provider immediately after + // the conditional delete; removing the owner here would orphan it. + const ownerConflict = await ensureLegacyOwner(redis, keys, input.keyId, ttlSeconds); + if (ownerConflict) return ownerConflict; + + const conflictAfterTerminate = await rejectLegacyMutationAfterCanonicalAppeared( + redis, + keys, + undefined, + { value: providerToTerminate.toString(), ttlSeconds } + ); + if (conflictAfterTerminate) return conflictAfterTerminate; + return { + status: "ok", + changed: true, + providerId: null, + terminatedProviderId: providerToTerminate, + }; + } + + if (legacyProvider !== null) { + const deleted = await deleteLegacyProviderIfValue( + redis, + keys.legacyProvider, + legacyProvider + ); + if (!deleted) { + const conflictAfterConcurrentMutation = + await rejectLegacyMutationAfterCanonicalAppeared(redis, keys); + if (conflictAfterConcurrentMutation) return conflictAfterConcurrentMutation; + return conflict("provider_mismatch"); + } + } + + // Keep the tenant owner as a null-binding tombstone. Besides matching + // the versioned terminate semantics, this prevents a recovered + // versioned worker from losing its required owner mirror while this + // fallback operation is in flight. + { + const ownerConflict = await ensureLegacyOwner(redis, keys, input.keyId, ttlSeconds); + if (ownerConflict) return ownerConflict; + } + { + const conflictAfterTerminate = await rejectLegacyMutationAfterCanonicalAppeared( + redis, + keys, + undefined, + legacyProvider === null ? undefined : { value: legacyProvider.toString(), ttlSeconds } + ); + if (conflictAfterTerminate) return conflictAfterTerminate; + } + return { status: "ok", changed: legacyProvider !== null, providerId: null }; + } + } catch (error) { + logger.warn("Legacy session binding mutation failed", { + error: error instanceof Error ? error.message : String(error), + }); + return unavailable("operation_failed"); + } +} + +export async function isSessionProviderCoolingDown( + input: SessionProviderCooldownInput +): Promise { + if ( + input.sessionId.length === 0 || + !isPositiveInteger(input.keyId) || + !isPositiveInteger(input.providerId) + ) { + return conflict("invalid_input"); + } + + const ready = await readyVersionedClient(input.redis); + if ("status" in ready) return ready; + + try { + const value = await ready.redis.get( + buildSessionProviderCooldownKey(input.sessionId, input.keyId, input.providerId) + ); + if (!connectionIsCurrent(ready)) return unavailable("connection_changed"); + return { status: "ok", coolingDown: value !== null, legacyFallbackAllowed: false }; + } catch (error) { + return handleOperationError(ready, error); + } +} + +export function resetVersionedBindingCapabilityForTests(): void { + detachCapabilityListeners(); + capabilityClient = null; + capabilityState = "unknown"; + capabilityEpoch = 0; + capabilityProbe = null; + capabilityListeners = null; +} diff --git a/src/lib/session-manager.ts b/src/lib/session-manager.ts index d44272252..f96169f0e 100644 --- a/src/lib/session-manager.ts +++ b/src/lib/session-manager.ts @@ -31,6 +31,28 @@ import { getKeyActiveSessionsKey, getUserActiveSessionsKey, } from "./redis/active-session-keys"; +import { + acquireSessionDiscoveryLease as acquireVersionedSessionDiscoveryLease, + buildSessionBindingKeys, + clearSessionBinding as clearVersionedSessionBinding, + compareAndSetSessionBinding, + ensureVersionedBindingCapability, + mutateLegacySessionBindingSafely, + readOrReconcileSessionBinding, + isSessionProviderCoolingDown as readSessionProviderCooldown, + getVersionedBindingCapabilityState as readVersionedBindingCapabilityState, + releaseSessionDiscoveryLease as releaseVersionedSessionDiscoveryLease, + renewSessionDiscoveryLease as renewVersionedSessionDiscoveryLease, + type SessionBindingResult, + type SessionBindingSnapshot, + type SessionBindingUnavailableResult, + type SessionDiscoveryLeaseAcquireResult, + type SessionDiscoveryLeaseMutationResult, + type SessionProviderCooldownResult, + terminateSessionBinding as terminateVersionedSessionBinding, + touchSessionBinding, + type VersionedBindingCapabilityState, +} from "./redis/session-binding"; import { SessionTracker } from "./session-tracker"; const RESERVED_INTERNAL_HEADER_SET = new Set( @@ -42,6 +64,15 @@ function isReservedInternalHeader(name: string): boolean { return lowerName.startsWith("x-cch-") || RESERVED_INTERNAL_HEADER_SET.has(lowerName); } +function redisUnavailableBindingResult(): SessionBindingUnavailableResult { + return { + status: "unavailable", + reason: "redis_not_ready", + capabilityState: readVersionedBindingCapabilityState(), + legacyFallbackAllowed: true, + }; +} + /** * 将已脱敏的 header 文本解析为可序列化对象（用于写入 Session 元信息）。 */ @@ -192,6 +223,10 @@ function parseSessionDetailResponseMeta(value: string): SessionDetailResponseMet } } +function buildTenantContentHashSessionKey(keyId: number, contentHash: string): string { + return `hash:${keyId}:${contentHash}:session`; +} + /** * Session 管理器 * @@ -283,6 +318,42 @@ export class SessionManager { return `sess_${timestamp}_${random}`; } + private static async proveContentHashSessionOwnership( + redis: NonNullable>, + sessionId: string, + keyId: number + ): Promise<{ owned: boolean; ownerPresent: boolean }> { + const legacyOwner = await redis.get(`session:${sessionId}:key`); + if (legacyOwner !== keyId.toString()) { + return { owned: false, ownerPresent: legacyOwner !== null }; + } + + // Reconcile only after the legacy owner proves the tenant. This both + // validates canonical/mirror consistency and refreshes the complete binding + // TTL, preventing an owner-expiry race between hash lookup and Provider selection. + const binding = await readOrReconcileSessionBinding({ + sessionId, + keyId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + }); + if (binding.status === "ok") { + return { owned: true, ownerPresent: true }; + } + if (!binding.legacyFallbackAllowed) { + return { owned: false, ownerPresent: true }; + } + + const legacyRefresh = await mutateLegacySessionBindingSafely({ + sessionId, + keyId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + mutation: { type: "refresh" }, + }); + return { owned: legacyRefresh.status === "ok", ownerPresent: true }; + } + /** * 获取 Session 内下一个请求序号（原子操作） * @@ -515,17 +586,32 @@ export class SessionManager { // 3. 尝试从 Redis 查找已有 session if (redis && redis.status === "ready") { try { - const hashKey = `hash:${contentHash}:session`; + const hashKey = buildTenantContentHashSessionKey(keyId, contentHash); const existingSessionId = await redis.get(hashKey); if (existingSessionId) { - // 找到已有 session，刷新 TTL - await SessionManager.refreshSessionTTL(existingSessionId, keyId); - logger.trace("SessionManager: Reusing session via hash", { - sessionId: existingSessionId, + const ownership = await SessionManager.proveContentHashSessionOwnership( + redis, + existingSessionId, + keyId + ); + if (ownership.owned) { + // 找到当前 tenant 的已有 session，刷新 TTL + await SessionManager.refreshSessionTTL(existingSessionId, keyId); + logger.trace("SessionManager: Reusing tenant-scoped session via hash", { + sessionId: existingSessionId, + hash: contentHash, + keyId, + }); + return existingSessionId; + } + + logger.warn("SessionManager: Ignoring content-hash mapping without matching owner", { hash: contentHash, + keyId, + mappingScope: "tenant", + ownerPresent: ownership.ownerPresent, }); - return existingSessionId; } // 未找到：创建新 session @@ -562,14 +648,34 @@ export class SessionManager { if (!redis || redis.status !== "ready") return; try { + const binding = await readOrReconcileSessionBinding({ + sessionId, + keyId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + }); + if (binding.status !== "ok") { + if (!binding.legacyFallbackAllowed) return; + const legacy = await mutateLegacySessionBindingSafely({ + sessionId, + keyId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + mutation: { type: "inspect" }, + }); + if (legacy.status !== "ok") return; + } + const pipeline = redis.pipeline(); - const hashKey = `hash:${contentHash}:session`; + // Do not dual-write the historical unscoped key. Mixed-version workers + // may temporarily create separate Sessions; old mappings expire naturally + // without allowing the new path to import tenant-ambiguous state. + const hashKey = buildTenantContentHashSessionKey(keyId, contentHash); // 存储映射关系 pipeline.setex(hashKey, SessionManager.SESSION_TTL, sessionId); - // 初始化 session 元数据 - pipeline.setex(`session:${sessionId}:key`, SessionManager.SESSION_TTL, keyId.toString()); + // Initialize non-binding session metadata after tenant ownership is proven. pipeline.setex( `session:${sessionId}:last_seen`, SessionManager.SESSION_TTL, @@ -587,16 +693,31 @@ export class SessionManager { /** * 刷新 session TTL（滑动窗口） */ - private static async refreshSessionTTL(sessionId: string, _keyId?: number | null): Promise { + private static async refreshSessionTTL(sessionId: string, keyId?: number | null): Promise { const redis = getRedisClient(); if (!redis || redis.status !== "ready") return; try { const pipeline = redis.pipeline(); - - // TTL 刷新不能改写 session 归属；这里只延长已有 key/provider 绑定的存活时间。 - pipeline.expire(`session:${sessionId}:key`, SessionManager.SESSION_TTL); - pipeline.expire(`session:${sessionId}:provider`, SessionManager.SESSION_TTL); + // Provider selection performs the authoritative binding reconcile. Keep + // this path limited to session activity metadata so a request does not + // pay for a second full binding Lua round trip before selection. + if (keyId != null && readVersionedBindingCapabilityState() === "unavailable") { + const legacyRefresh = await mutateLegacySessionBindingSafely({ + sessionId, + keyId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + mutation: { type: "refresh" }, + }); + if (legacyRefresh.status !== "ok") { + logger.warn("SessionManager: Legacy binding TTL refresh blocked", { + sessionId, + keyId, + reason: legacyRefresh.reason, + }); + } + } pipeline.setex( `session:${sessionId}:last_seen`, SessionManager.SESSION_TTL, @@ -609,6 +730,144 @@ export class SessionManager { } } + static getVersionedBindingCapabilityState(): VersionedBindingCapabilityState { + return readVersionedBindingCapabilityState(); + } + + static async ensureVersionedBindingCapability(): Promise { + return ensureVersionedBindingCapability(); + } + + static async acquireSessionDiscoveryLease( + sessionId: string, + keyId: number, + ttlSeconds: number, + ownerToken?: string + ): Promise { + const redis = getRedisClient(); + if (!redis || redis.status !== "ready") return redisUnavailableBindingResult(); + return acquireVersionedSessionDiscoveryLease({ + sessionId, + keyId, + ttlSeconds, + ownerToken, + redis, + }); + } + + static async renewSessionDiscoveryLease( + sessionId: string, + keyId: number, + ownerToken: string, + ttlSeconds: number + ): Promise { + const redis = getRedisClient(); + if (!redis || redis.status !== "ready") return redisUnavailableBindingResult(); + return renewVersionedSessionDiscoveryLease({ + sessionId, + keyId, + ownerToken, + ttlSeconds, + redis, + }); + } + + static async releaseSessionDiscoveryLease( + sessionId: string, + keyId: number, + ownerToken: string + ): Promise { + const redis = getRedisClient(); + if (!redis || redis.status !== "ready") return redisUnavailableBindingResult(); + return releaseVersionedSessionDiscoveryLease({ sessionId, keyId, ownerToken, redis }); + } + + static async getSessionBindingSnapshot( + sessionId: string, + keyId: number + ): Promise { + const redis = getRedisClient(); + if (!redis || redis.status !== "ready") return redisUnavailableBindingResult(); + return readOrReconcileSessionBinding({ + sessionId, + keyId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + }); + } + + /** + * Heartbeats run at one third of the configured binding TTL, leaving time + * for a transient Redis failure without allowing a live binding to expire. + */ + static getVersionedSessionBindingRefreshIntervalMs(): number { + return Math.max(1, Math.floor((SessionManager.SESSION_TTL * 1000) / 3)); + } + + static async touchVersionedSessionBinding( + snapshot: SessionBindingSnapshot + ): Promise { + const redis = getRedisClient(); + if (!redis || redis.status !== "ready") return redisUnavailableBindingResult(); + return touchSessionBinding({ + sessionId: snapshot.sessionId, + keyId: snapshot.keyId, + expectedGeneration: snapshot.generation, + expectedProviderId: snapshot.providerId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + }); + } + + static async compareAndSetSessionProvider( + snapshot: SessionBindingSnapshot, + providerId: number + ): Promise { + const redis = getRedisClient(); + if (!redis || redis.status !== "ready") return redisUnavailableBindingResult(); + return compareAndSetSessionBinding({ + sessionId: snapshot.sessionId, + keyId: snapshot.keyId, + expectedGeneration: snapshot.generation, + providerId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + }); + } + + static async clearVersionedSessionProvider( + snapshot: SessionBindingSnapshot, + expectedProviderId: number | null, + cooldownTtlSeconds: number = 0 + ): Promise { + const redis = getRedisClient(); + if (!redis || redis.status !== "ready") return redisUnavailableBindingResult(); + return clearVersionedSessionBinding({ + sessionId: snapshot.sessionId, + keyId: snapshot.keyId, + expectedGeneration: snapshot.generation, + expectedProviderId, + cooldownTtlSeconds, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + }); + } + + static async isSessionProviderCoolingDown( + sessionId: string, + keyId: number, + providerId: number + ): Promise { + const redis = getRedisClient(); + if (!redis || redis.status !== "ready") return redisUnavailableBindingResult(); + return readSessionProviderCooldown({ + sessionId, + keyId, + providerId, + redis, + }); + } + /** * 绑定 session 到 provider（TC-009 修复：使用 SET NX 避免竞态条件） */ @@ -621,35 +880,72 @@ export class SessionManager { if (!redis || redis.status !== "ready") return; try { - const key = `session:${sessionId}:provider`; - // 使用 SET ... NX 保证只有第一次绑定成功（原子操作） - const result = await redis.set( - key, - providerId.toString(), - "EX", - SessionManager.SESSION_TTL, - "NX" // Only set if not exists - ); - - if (result === "OK") { - if (keyId != null) { - await redis.setex( - `session:${sessionId}:key`, - SessionManager.SESSION_TTL, - keyId.toString() - ); - } - logger.trace("SessionManager: Bound session to provider", { + if (keyId != null) { + const binding = await readOrReconcileSessionBinding({ sessionId, - providerId, + keyId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, }); - } else { - // 已绑定过，不覆盖（避免并发请求选择不同供应商） - logger.debug("SessionManager: Session already bound, skipping", { + if (binding.status === "ok") { + if (binding.snapshot.providerId !== null) { + logger.debug("SessionManager: Session already bound, skipping", { + sessionId, + attemptedProviderId: providerId, + }); + return; + } + + const updated = await compareAndSetSessionBinding({ + sessionId, + keyId, + expectedGeneration: binding.snapshot.generation, + providerId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + }); + if (updated.status === "ok") { + logger.trace("SessionManager: Bound versioned session to provider", { + sessionId, + providerId, + }); + } + return; + } + if (!binding.legacyFallbackAllowed) { + logger.warn("SessionManager: Versioned session binding is not writable", { + sessionId, + keyId, + reason: binding.reason, + }); + return; + } + const legacy = await mutateLegacySessionBindingSafely({ sessionId, - attemptedProviderId: providerId, + keyId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + mutation: { type: "bind_if_absent", providerId }, }); + if (legacy.status === "ok" && legacy.changed) { + logger.trace("SessionManager: Bound legacy session to provider", { + sessionId, + providerId, + }); + } else if (legacy.status !== "ok") { + logger.warn("SessionManager: Legacy session binding blocked", { + sessionId, + keyId, + reason: legacy.reason, + }); + } + return; } + + logger.warn("SessionManager: Cannot bind session without an API key owner", { + sessionId, + providerId, + }); } catch (error) { logger.error("SessionManager: Failed to bind provider", { error }); } @@ -667,6 +963,36 @@ export class SessionManager { try { if (keyId != null) { + const binding = await readOrReconcileSessionBinding({ + sessionId, + keyId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + }); + if (binding.status === "ok") { + return binding.snapshot.providerId; + } + if (!binding.legacyFallbackAllowed) { + logger.warn("SessionManager: Versioned session binding is unavailable for reuse", { + sessionId, + keyId, + reason: binding.reason, + }); + return null; + } + + // A capability failure may permit a legacy read only when this + // session has no canonical binding. If canonical state exists, the + // legacy mirror is not safely writable and must not be reused. + const bindingKeys = buildSessionBindingKeys(sessionId, keyId); + if ((await redis.exists(bindingKeys.canonical)) > 0) { + logger.warn("SessionManager: Refusing legacy provider reuse with canonical binding", { + sessionId, + keyId, + }); + return null; + } + const boundKeyId = await redis.get(`session:${sessionId}:key`); // Fail-closed：boundKeyId 缺失（TTL 漂移、旧绑定或写入路径未原子写 key）也视为校验失败， // 避免无法证明归属当前 key 的旧 provider binding 继续被复用。 @@ -699,41 +1025,123 @@ export class SessionManager { */ static async clearSessionProvider( sessionId: string, - expectedProviderId?: number | null + expectedProviderId?: number | null, + keyId?: number | null ): Promise { const redis = getRedisClient(); if (!redis || redis.status !== "ready") return false; try { - const key = `session:${sessionId}:provider`; - const deleted = - expectedProviderId == null - ? await redis.del(key) - : Number( - await redis.eval( - ` - if redis.call("GET", KEYS[1]) == ARGV[1] then - return redis.call("DEL", KEYS[1]) - end - return 0 - `, - 1, - key, - expectedProviderId.toString() - ) - ); - logger.trace("SessionManager: Cleared session provider binding", { + if (keyId != null) { + const binding = await readOrReconcileSessionBinding({ + sessionId, + keyId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + }); + if (binding.status === "ok") { + const currentProviderId = binding.snapshot.providerId; + if ( + currentProviderId === null || + (expectedProviderId != null && currentProviderId !== expectedProviderId) + ) { + return false; + } + + const cleared = await clearVersionedSessionBinding({ + sessionId, + keyId, + expectedGeneration: binding.snapshot.generation, + expectedProviderId: currentProviderId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + }); + const didClear = cleared.status === "ok"; + logger.trace("SessionManager: Cleared versioned session provider binding", { + sessionId, + keyId, + expectedProviderId: expectedProviderId ?? null, + deleted: didClear, + }); + return didClear; + } + if (!binding.legacyFallbackAllowed) { + logger.warn("SessionManager: Versioned session binding clear blocked", { + sessionId, + keyId, + reason: binding.reason, + }); + return false; + } + const legacy = await mutateLegacySessionBindingSafely({ + sessionId, + keyId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + mutation: { type: "clear", expectedProviderId }, + }); + return legacy.status === "ok" && legacy.changed; + } + + logger.warn("SessionManager: Cannot clear session binding without an API key owner", { sessionId, expectedProviderId: expectedProviderId ?? null, - deleted: deleted > 0, }); - return deleted > 0; + return false; } catch (error) { logger.error("SessionManager: Failed to clear session provider", { error, sessionId }); return false; } } + static async clearSessionProviders( + sessionId: string, + expectedProviderIds: Iterable, + keyId?: number | null + ): Promise { + const providerIds = Array.from( + new Set( + Array.from(expectedProviderIds).filter( + (providerId) => Number.isSafeInteger(providerId) && providerId > 0 + ) + ) + ); + if (providerIds.length === 0 || keyId == null) return false; + + const redis = getRedisClient(); + if (!redis || redis.status !== "ready") return false; + + const binding = await readOrReconcileSessionBinding({ + sessionId, + keyId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + }); + if (binding.status === "ok") { + const providerId = binding.snapshot.providerId; + if (providerId === null || !providerIds.includes(providerId)) return false; + const cleared = await clearVersionedSessionBinding({ + sessionId, + keyId, + expectedGeneration: binding.snapshot.generation, + expectedProviderId: providerId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + }); + return cleared.status === "ok"; + } + if (!binding.legacyFallbackAllowed) return false; + + const legacy = await mutateLegacySessionBindingSafely({ + sessionId, + keyId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + mutation: { type: "clear", expectedProviderIds: providerIds }, + }); + return legacy.status === "ok" && legacy.changed; + } + /** * 获取当前绑定供应商的优先级 * @@ -743,19 +1151,16 @@ export class SessionManager { * @param sessionId - Session ID * @returns 优先级数字（数字越小优先级越高），如果未绑定或无法查询则返回 null */ - static async getSessionProviderPriority(sessionId: string): Promise { + static async getSessionProviderPriority( + sessionId: string, + keyId?: number | null + ): Promise { const redis = getRedisClient(); if (!redis || redis.status !== "ready") return null; try { - // 修复：从真实绑定关系读取（session:provider） - const providerIdStr = await redis.get(`session:${sessionId}:provider`); - if (!providerIdStr) { - return null; - } - - const providerId = parseInt(providerIdStr, 10); - if (Number.isNaN(providerId)) { + const providerId = await SessionManager.getSessionProvider(sessionId, keyId); + if (providerId === null) { return null; } @@ -780,7 +1185,8 @@ export class SessionManager { /** * 智能更新 Session 绑定 * - * 策略：首次绑定用 SET NX；故障转移成功或竞速赢家强制改绑时无条件更新；其他情况按优先级和熔断状态决策 + * 策略：首次绑定用条件创建；故障转移成功或竞速赢家跳过优先级/熔断决策， + * 但版本化路径仍以读取到的 generation 做 CAS，避免迟到请求覆盖更新的绑定。 */ static async updateSessionBindingSmart( sessionId: string, @@ -790,33 +1196,109 @@ export class SessionManager { isFailoverSuccess: boolean = false, keyId?: number | null, forceUpdate: boolean = false - ): Promise<{ updated: boolean; reason: string; details?: string }> { + ): Promise<{ + updated: boolean; + reason: string; + details?: string; + bindingSnapshot?: SessionBindingSnapshot; + legacyBindingUpdated?: boolean; + }> { const redis = getRedisClient(); if (!redis || redis.status !== "ready") { return { updated: false, reason: "redis_not_ready" }; } try { - // ========== 情况 1：首次尝试成功 ========== - if (isFirstAttempt) { - const key = `session:${sessionId}:provider`; - // 使用 SET NX 绑定（避免覆盖并发请求） - const result = await redis.set( - key, - newProviderId.toString(), - "EX", - SessionManager.SESSION_TTL, - "NX" - ); + let versionedSnapshot: SessionBindingSnapshot | null = null; + let committedVersionedSnapshot: SessionBindingSnapshot | null = null; + let committedLegacyBinding = false; + let useLegacyBinding = false; + let legacyProviderId: number | null = null; - if (result === "OK") { - if (keyId != null) { - await redis.setex( - `session:${sessionId}:key`, - SessionManager.SESSION_TTL, - keyId.toString() - ); + if (keyId != null) { + const binding = await readOrReconcileSessionBinding({ + sessionId, + keyId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + }); + if (binding.status === "ok") { + versionedSnapshot = binding.snapshot; + } else if (binding.legacyFallbackAllowed) { + const legacy = await mutateLegacySessionBindingSafely({ + sessionId, + keyId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + mutation: { type: "inspect" }, + }); + if (legacy.status !== "ok") { + return { + updated: false, + reason: "legacy_binding_conflict", + details: legacy.reason, + }; + } + useLegacyBinding = true; + legacyProviderId = legacy.providerId; + } else { + return { + updated: false, + reason: "versioned_binding_conflict", + details: binding.reason, + }; + } + } else { + return { + updated: false, + reason: "binding_owner_unavailable", + details: "Cannot mutate a Session binding without an API key owner", + }; + } + + const persistBinding = async (onlyIfUnbound: boolean): Promise => { + if (versionedSnapshot) { + if (onlyIfUnbound && versionedSnapshot.providerId !== null) { + return false; + } + const result = await compareAndSetSessionBinding({ + sessionId, + keyId: versionedSnapshot.keyId, + expectedGeneration: versionedSnapshot.generation, + providerId: newProviderId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + }); + if (result.status !== "ok") { + logger.warn("SessionManager: Versioned session binding CAS did not update", { + sessionId, + keyId: versionedSnapshot.keyId, + providerId: newProviderId, + reason: result.reason, + }); + return false; } + committedVersionedSnapshot = result.snapshot; + return true; + } + + if (!useLegacyBinding) return false; + const result = await mutateLegacySessionBindingSafely({ + sessionId, + keyId: keyId!, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + mutation: onlyIfUnbound + ? { type: "bind_if_absent", providerId: newProviderId } + : { type: "set", providerId: newProviderId }, + }); + const updated = result.status === "ok" && result.changed; + if (updated) committedLegacyBinding = true; + return updated; + }; + + if (isFirstAttempt) { + if (await persistBinding(true)) { logger.info("SessionManager: Bound session to provider (first success)", { sessionId, providerId: newProviderId, @@ -827,31 +1309,23 @@ export class SessionManager { reason: "first_success", details: `首次成功，绑定到供应商 ${newProviderId} (priority=${newProviderPriority})`, }; - } else { - // 并发请求已经绑定了，放弃更新 - return { - updated: false, - reason: "concurrent_binding_exists", - details: "并发请求已绑定，跳过", - }; } + return { + updated: false, + reason: "concurrent_binding_exists", + details: "并发请求已绑定，跳过", + }; } - // ========== 情况 2：重试成功（需要智能决策）========== - - // 2.0 故障转移成功 或 竞速赢家强制改绑：无条件更新绑定 - // forceUpdate 在读取当前绑定/优先级/熔断状态之前短路，确保竞速赢家一定成为复用绑定。 if (isFailoverSuccess || forceUpdate) { - const pipeline = redis.pipeline(); - pipeline.setex( - `session:${sessionId}:provider`, - SessionManager.SESSION_TTL, - newProviderId.toString() - ); - if (keyId != null) { - pipeline.setex(`session:${sessionId}:key`, SessionManager.SESSION_TTL, keyId.toString()); + const updated = await persistBinding(false); + if (!updated) { + return { + updated: false, + reason: "concurrent_binding_changed", + details: "Session binding changed before the update committed", + }; } - await pipeline.exec(); const reason = isFailoverSuccess ? "failover_success" : "race_winner_forced"; logger.info( @@ -871,30 +1345,15 @@ export class SessionManager { details: isFailoverSuccess ? `故障转移成功，绑定到供应商 ${newProviderId}` : `竞速赢家强制改绑到供应商 ${newProviderId}`, + ...(committedVersionedSnapshot ? { bindingSnapshot: committedVersionedSnapshot } : {}), + ...(committedLegacyBinding ? { legacyBindingUpdated: true } : {}), }; } - // 2.1 获取当前绑定的供应商 ID - const currentProviderIdStr = await redis.get(`session:${sessionId}:provider`); - if (!currentProviderIdStr) { - // 没有绑定，使用 SET NX 绑定 - const key = `session:${sessionId}:provider`; - const result = await redis.set( - key, - newProviderId.toString(), - "EX", - SessionManager.SESSION_TTL, - "NX" - ); + const currentProviderId: number | null = versionedSnapshot?.providerId ?? legacyProviderId; - if (result === "OK") { - if (keyId != null) { - await redis.setex( - `session:${sessionId}:key`, - SessionManager.SESSION_TTL, - keyId.toString() - ); - } + if (currentProviderId === null) { + if (await persistBinding(true)) { logger.info("SessionManager: Bound session (no previous binding)", { sessionId, providerId: newProviderId, @@ -905,39 +1364,21 @@ export class SessionManager { reason: "no_previous_binding", details: `无绑定，绑定到供应商 ${newProviderId} (priority=${newProviderPriority})`, }; - } else { - return { - updated: false, - reason: "concurrent_binding_exists", - details: "并发请求已绑定", - }; } + return { + updated: false, + reason: "concurrent_binding_exists", + details: "并发请求已绑定", + }; } - const currentProviderId = parseInt(currentProviderIdStr, 10); - if (Number.isNaN(currentProviderId)) { - logger.warn("SessionManager: Invalid provider ID in Redis", { - currentProviderIdStr, - }); - return { updated: false, reason: "invalid_provider_id" }; - } - - // 2.2 查询当前供应商的详情（优先级 + 健康状态） const { findProviderById } = await import("@/repository/provider"); const currentProvider = await findProviderById(currentProviderId); if (!currentProvider) { - // 当前供应商不存在（可能被删除），直接更新 - const pipeline = redis.pipeline(); - pipeline.setex( - `session:${sessionId}:provider`, - SessionManager.SESSION_TTL, - newProviderId.toString() - ); - if (keyId != null) { - pipeline.setex(`session:${sessionId}:key`, SessionManager.SESSION_TTL, keyId.toString()); + if (!(await persistBinding(false))) { + return { updated: false, reason: "concurrent_binding_changed" }; } - await pipeline.exec(); logger.info("SessionManager: Updated binding (current provider not found)", { sessionId, @@ -955,20 +1396,10 @@ export class SessionManager { const currentPriority = currentProvider.priority || 0; - // 2.3 智能决策：优先级比较 + 健康检查 - - // ========== 规则 A：新供应商优先级更高（数字更小）→ 直接迁移 ========== if (newProviderPriority < currentPriority) { - const pipeline = redis.pipeline(); - pipeline.setex( - `session:${sessionId}:provider`, - SessionManager.SESSION_TTL, - newProviderId.toString() - ); - if (keyId != null) { - pipeline.setex(`session:${sessionId}:key`, SessionManager.SESSION_TTL, keyId.toString()); + if (!(await persistBinding(false))) { + return { updated: false, reason: "concurrent_binding_changed" }; } - await pipeline.exec(); logger.info("SessionManager: Migrated to higher priority provider", { sessionId, @@ -986,22 +1417,13 @@ export class SessionManager { }; } - // ========== 规则 B：新供应商优先级相同或更低 → 检查原供应商健康状态 ========== const { isCircuitOpen } = await import("@/lib/circuit-breaker"); const isCurrentCircuitOpen = await isCircuitOpen(currentProviderId); if (isCurrentCircuitOpen) { - // 原供应商已熔断 → 更新到新供应商（备用供应商接管） - const pipeline = redis.pipeline(); - pipeline.setex( - `session:${sessionId}:provider`, - SessionManager.SESSION_TTL, - newProviderId.toString() - ); - if (keyId != null) { - pipeline.setex(`session:${sessionId}:key`, SessionManager.SESSION_TTL, keyId.toString()); + if (!(await persistBinding(false))) { + return { updated: false, reason: "concurrent_binding_changed" }; } - await pipeline.exec(); logger.info("SessionManager: Migrated to backup provider (circuit open)", { sessionId, @@ -1019,7 +1441,6 @@ export class SessionManager { }; } - // 原供应商健康 + 优先级更高/相同 → 保持原绑定（尽量使用主供应商） logger.debug("SessionManager: Keeping current provider (healthy and higher/equal priority)", { sessionId, currentProviderId, @@ -2362,52 +2783,91 @@ export class SessionManager { // 使用 prompt_cache_key 作为新的 Session ID（添加前缀以区分） const codexSessionId = `codex_${promptCacheKey}`; - // 检查是否已经存在绑定 - const existingProvider = await redis.get(`session:${codexSessionId}:provider`); + if (keyId != null) { + const binding = await readOrReconcileSessionBinding({ + sessionId: codexSessionId, + keyId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + }); + if (binding.status === "ok") { + if (binding.snapshot.providerId !== null) { + logger.debug("SessionManager: Refreshed versioned Codex session TTL", { + sessionId: codexSessionId, + providerId: binding.snapshot.providerId, + }); + return { sessionId: codexSessionId, updated: false }; + } - if (existingProvider) { - // 已存在绑定，刷新 TTL - const pipeline = redis.pipeline(); - pipeline.expire(`session:${codexSessionId}:provider`, SessionManager.SESSION_TTL); - if (keyId != null) { - pipeline.setex( - `session:${codexSessionId}:key`, - SessionManager.SESSION_TTL, - keyId.toString() - ); + const updated = await compareAndSetSessionBinding({ + sessionId: codexSessionId, + keyId, + expectedGeneration: binding.snapshot.generation, + providerId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + }); + if (updated.status === "ok") { + logger.info("SessionManager: Created versioned Codex session", { + sessionId: codexSessionId, + providerId, + }); + return { sessionId: codexSessionId, updated: true }; + } + return { sessionId: currentSessionId, updated: false }; } - await pipeline.exec(); - logger.debug("SessionManager: Refreshed Codex session TTL", { + if (!binding.legacyFallbackAllowed) { + logger.warn("SessionManager: Codex session binding owner could not be verified", { + sessionId: codexSessionId, + keyId, + reason: binding.reason, + }); + return { sessionId: currentSessionId, updated: false }; + } + const legacy = await mutateLegacySessionBindingSafely({ sessionId: codexSessionId, - providerId: parseInt(existingProvider, 10), + keyId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + mutation: { type: "inspect" }, }); - return { sessionId: codexSessionId, updated: false }; - } + if (legacy.status !== "ok") { + logger.warn("SessionManager: Legacy Codex binding owner could not be verified", { + sessionId: codexSessionId, + keyId, + reason: legacy.reason, + }); + return { sessionId: currentSessionId, updated: false }; + } - // 新建绑定 - const pipeline = redis.pipeline(); - pipeline.setex( - `session:${codexSessionId}:provider`, - SessionManager.SESSION_TTL, - providerId.toString() - ); - if (keyId != null) { - pipeline.setex( - `session:${codexSessionId}:key`, - SessionManager.SESSION_TTL, - keyId.toString() - ); + if (legacy.providerId !== null) { + await mutateLegacySessionBindingSafely({ + sessionId: codexSessionId, + keyId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + mutation: { type: "refresh" }, + }); + return { sessionId: codexSessionId, updated: false }; + } + + const bound = await mutateLegacySessionBindingSafely({ + sessionId: codexSessionId, + keyId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + mutation: { type: "bind_if_absent", providerId }, + }); + if (bound.status === "ok") { + return { sessionId: codexSessionId, updated: bound.changed }; + } + return { sessionId: currentSessionId, updated: false }; } - await pipeline.exec(); - logger.info("SessionManager: Created Codex session from prompt_cache_key", { + logger.warn("SessionManager: Cannot bind Codex session without an API key owner", { sessionId: codexSessionId, - promptCacheKey, - providerId, - ttl: SessionManager.SESSION_TTL, }); - - return { sessionId: codexSessionId, updated: true }; + return { sessionId: currentSessionId, updated: false }; } catch (error) { logger.error("SessionManager: Failed to update Codex session", { error }); return { sessionId: currentSessionId, updated: false }; @@ -2423,7 +2883,10 @@ export class SessionManager { * @param sessionId - Session ID * @returns 是否成功删除 */ - static async terminateSession(sessionId: string): Promise { + static async terminateSession( + sessionId: string, + expectedProviderIds?: readonly number[] + ): Promise { const redis = getRedisClient(); if (!redis || redis.status !== "ready") { logger.warn("SessionManager: Redis not ready, cannot terminate session"); @@ -2435,6 +2898,7 @@ export class SessionManager { let providerId: number | null = null; let keyId: number | null = null; let userId: number | null = null; + let bindingTerminated = false; try { const [providerIdStr, keyIdStr, userIdStr] = await Promise.all([ @@ -2443,11 +2907,17 @@ export class SessionManager { redis.hget(`session:${sessionId}:info`, "userId"), ]); - providerId = providerIdStr ? parseInt(providerIdStr, 10) : null; - keyId = keyIdStr ? parseInt(keyIdStr, 10) : null; - userId = userIdStr ? parseInt(userIdStr, 10) : null; + providerId = providerIdStr ? Number(providerIdStr) : null; + keyId = keyIdStr ? Number(keyIdStr) : null; + userId = userIdStr ? Number(userIdStr) : null; - if (!Number.isFinite(userId)) { + if (providerId !== null && (!Number.isSafeInteger(providerId) || providerId <= 0)) { + providerId = null; + } + if (keyId !== null && (!Number.isSafeInteger(keyId) || keyId <= 0)) { + keyId = null; + } + if (userId !== null && (!Number.isSafeInteger(userId) || userId <= 0)) { userId = null; } } catch (lookupError) { @@ -2461,12 +2931,167 @@ export class SessionManager { ); } + if (keyId !== null) { + const binding = await readOrReconcileSessionBinding({ + sessionId, + keyId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + }); + if (binding.status === "ok") { + providerId = binding.snapshot.providerId ?? providerId; + if ( + expectedProviderIds && + (binding.snapshot.providerId === null || + !expectedProviderIds.includes(binding.snapshot.providerId)) + ) { + return false; + } + + const terminated = await terminateVersionedSessionBinding({ + sessionId, + keyId, + expectedProviderId: expectedProviderIds + ? (binding.snapshot.providerId ?? undefined) + : undefined, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + }); + if (terminated.status !== "ok") { + logger.warn("SessionManager: Versioned session termination blocked", { + sessionId, + keyId, + reason: terminated.reason, + }); + return false; + } + + if (expectedProviderIds) { + const terminatedProviderId = binding.snapshot.providerId; + if (terminatedProviderId === null) { + logger.warn("SessionManager: Scoped versioned termination lost Provider identity", { + sessionId, + keyId, + }); + return false; + } + + // The versioned CAS above is the linearization point. A failover + // may bind this Session to Q immediately afterwards, so scoped + // invalidation must only remove P's Provider-owned indexes. + try { + const providerCleanup = redis.pipeline(); + providerCleanup.zrem(`provider:${terminatedProviderId}:active_sessions`, sessionId); + providerCleanup.hdel( + `provider:${terminatedProviderId}:active_session_refs`, + sessionId + ); + await providerCleanup.exec(); + } catch (cleanupError) { + logger.warn("SessionManager: Scoped versioned Provider index cleanup failed", { + sessionId, + providerId: terminatedProviderId, + error: cleanupError, + }); + } + + logger.info("SessionManager: Cleared scoped versioned Provider binding", { + sessionId, + providerId: terminatedProviderId, + keyId, + }); + return true; + } + + bindingTerminated = true; + } else if (binding.status === "unavailable" && binding.legacyFallbackAllowed) { + const legacy = await mutateLegacySessionBindingSafely({ + sessionId, + keyId, + ttlSeconds: SessionManager.SESSION_TTL, + redis, + mutation: { type: "terminate", expectedProviderIds }, + }); + if (legacy.status !== "ok") { + logger.warn("SessionManager: Legacy session termination blocked", { + sessionId, + keyId, + reason: legacy.reason, + }); + return false; + } + + if (expectedProviderIds) { + const terminatedProviderId = legacy.terminatedProviderId; + if (terminatedProviderId == null) { + logger.warn("SessionManager: Scoped legacy termination lost Provider identity", { + sessionId, + keyId, + }); + return false; + } + + // The helper's value-checked delete is the linearization point. A + // failover may bind this Session to Q immediately afterwards, so + // provider-scoped invalidation must not delete shared Session + // metadata or global/key/user indexes after removing P. + try { + const providerCleanup = redis.pipeline(); + providerCleanup.zrem(`provider:${terminatedProviderId}:active_sessions`, sessionId); + providerCleanup.hdel( + `provider:${terminatedProviderId}:active_session_refs`, + sessionId + ); + await providerCleanup.exec(); + } catch (cleanupError) { + logger.warn("SessionManager: Scoped legacy Provider index cleanup failed", { + sessionId, + providerId: terminatedProviderId, + error: cleanupError, + }); + } + + logger.info("SessionManager: Cleared scoped legacy Provider binding", { + sessionId, + providerId: terminatedProviderId, + keyId, + }); + return true; + } + + bindingTerminated = true; + } else { + logger.warn("SessionManager: Session binding termination blocked", { + sessionId, + keyId, + reason: binding.reason, + }); + return false; + } + } else if (providerId !== null || expectedProviderIds) { + logger.warn("SessionManager: Session binding owner unavailable during termination", { + sessionId, + providerId, + }); + return false; + } + + // A binding-aware termination must succeed before any session metadata or + // active-session indexes are removed. This guard keeps a future mutation + // path from turning a CAS/mirror conflict into a misleading success based + // only on unrelated metadata deletions. + if (keyId !== null && !bindingTerminated) { + logger.warn("SessionManager: Session binding was not terminated", { + sessionId, + keyId, + }); + return false; + } + // 2. 删除所有 Session 相关的 key const pipeline = redis.pipeline(); - // 基础绑定信息 - pipeline.del(`session:${sessionId}:provider`); - pipeline.del(`session:${sessionId}:key`); + // Binding mirrors are mutated only by the tenant-authorized helpers above. pipeline.del(`session:${sessionId}:info`); pipeline.del(`session:${sessionId}:last_seen`); pipeline.del(`session:${sessionId}:concurrent_count`); @@ -2514,7 +3139,7 @@ export class SessionManager { deletedKeys, }); - return deletedKeys > 0; + return bindingTerminated || deletedKeys > 0; } catch (error) { logger.error("SessionManager: Failed to terminate session", { error, @@ -2572,7 +3197,10 @@ export class SessionManager { return 0; } - const terminatedCount = await SessionManager.terminateSessionsBatch([...sessionIds]); + const terminatedCount = await SessionManager.terminateSessionsBatch( + [...sessionIds], + uniqueProviderIds + ); logger.info("SessionManager: Terminated provider sessions batch", { providerIds: uniqueProviderIds, sessionCount: sessionIds.size, @@ -2617,7 +3245,10 @@ export class SessionManager { * @param sessionIds - Session ID 列表 * @returns 成功终止的数量 */ - static async terminateSessionsBatch(sessionIds: string[]): Promise { + static async terminateSessionsBatch( + sessionIds: string[], + expectedProviderIds?: readonly number[] + ): Promise { if (sessionIds.length === 0) { return 0; } @@ -2637,7 +3268,7 @@ export class SessionManager { const chunk = sessionIds.slice(i, i + CHUNK_SIZE); const results = await Promise.all( chunk.map(async (sessionId) => { - const success = await SessionManager.terminateSession(sessionId); + const success = await SessionManager.terminateSession(sessionId, expectedProviderIds); return success ? 1 : 0; }) ); diff --git a/src/lib/session-tracker.ts b/src/lib/session-tracker.ts index dd278a521..878d0fc98 100644 --- a/src/lib/session-tracker.ts +++ b/src/lib/session-tracker.ts @@ -5,6 +5,10 @@ import { getUserActiveSessionsKey, } from "@/lib/redis/active-session-keys"; import { getRedisClient } from "./redis"; +import { + getVersionedBindingCapabilityState, + mutateLegacySessionBindingSafely, +} from "./redis/session-binding"; const PROVIDER_ACTIVE_SESSIONS_PATTERN = /^provider:(\d+):active_sessions$/; @@ -197,8 +201,24 @@ export class SessionTracker { try { const now = Date.now(); - const pipeline = redis.pipeline(); const ttlSeconds = SessionTracker.SESSION_TTL_SECONDS; + if (getVersionedBindingCapabilityState() === "unavailable") { + const legacyRefresh = await mutateLegacySessionBindingSafely({ + sessionId, + keyId, + ttlSeconds, + redis, + mutation: { type: "refresh" }, + }); + if (legacyRefresh.status !== "ok") { + logger.warn("SessionTracker: Legacy binding TTL refresh blocked", { + sessionId, + keyId, + reason: legacyRefresh.reason, + }); + } + } + const pipeline = redis.pipeline(); const providerZSetKey = `provider:${providerId}:active_sessions`; const providerRefKey = `provider:${providerId}:active_session_refs`; const globalKey = getGlobalActiveSessionsKey(); @@ -222,10 +242,6 @@ export class SessionTracker { commandIndex++; } - pipeline.expire(`session:${sessionId}:provider`, ttlSeconds); - commandIndex++; - pipeline.expire(`session:${sessionId}:key`, ttlSeconds); - commandIndex++; pipeline.setex(`session:${sessionId}:last_seen`, ttlSeconds, now.toString()); commandIndex++; diff --git a/tests/configs/integration.config.ts b/tests/configs/integration.config.ts index ca2152353..58b4c0de3 100644 --- a/tests/configs/integration.config.ts +++ b/tests/configs/integration.config.ts @@ -10,6 +10,7 @@ export default createTestRunnerConfig({ "tests/integration/my-usage-imported-ledger.test.ts", "tests/integration/rolling-cost-redis.test.ts", "tests/integration/lease-settlement-redis.test.ts", + "tests/integration/session-binding-versioning-redis.test.ts", "tests/integration/db-pool-isolation-postgres.test.ts", "tests/integration/db-pool-slow-close-postgres.test.ts", "tests/integration/message-write-buffer-recovery-postgres.test.ts", diff --git a/tests/configs/session-binding.config.ts b/tests/configs/session-binding.config.ts new file mode 100644 index 000000000..0815cba93 --- /dev/null +++ b/tests/configs/session-binding.config.ts @@ -0,0 +1,9 @@ +import { createCoverageConfig } from "../vitest.base"; + +export default createCoverageConfig({ + name: "session-binding", + environment: "node", + testFiles: ["tests/unit/lib/redis/session-binding.test.ts"], + sourceFiles: ["src/lib/redis/session-binding.ts"], + thresholds: { lines: 80, functions: 80, branches: 75, statements: 80 }, +}); diff --git a/tests/integration/session-binding-versioning-redis.test.ts b/tests/integration/session-binding-versioning-redis.test.ts new file mode 100644 index 000000000..3459f7dba --- /dev/null +++ b/tests/integration/session-binding-versioning-redis.test.ts @@ -0,0 +1,593 @@ +import { randomUUID } from "node:crypto"; +import Redis from "ioredis"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "vitest"; +import { + buildCanonicalSessionBindingKey, + buildLegacySessionOwnerKey, + buildLegacySessionProviderKey, + buildSessionBindingKeys, + buildSessionProviderCooldownKey, + clearSessionBinding, + compareAndSetSessionBinding, + ensureVersionedBindingCapability, + isSessionProviderCoolingDown, + readOrReconcileSessionBinding, + resetVersionedBindingCapabilityForTests, + terminateSessionBinding, + touchSessionBinding, + type SessionBindingOkResult, + type SessionBindingResult, +} from "@/lib/redis/session-binding"; + +const HAS_REDIS = Boolean(process.env.REDIS_URL); +const EXPECTED_CAPABILITY_RAW = process.env.EXPECT_VERSIONED_BINDING_CAPABILITY; + +if ( + EXPECTED_CAPABILITY_RAW !== undefined && + EXPECTED_CAPABILITY_RAW !== "available" && + EXPECTED_CAPABILITY_RAW !== "unavailable" +) { + throw new Error("EXPECT_VERSIONED_BINDING_CAPABILITY must be either available or unavailable"); +} + +const EXPECTED_CAPABILITY = EXPECTED_CAPABILITY_RAW ?? "available"; +const runWithRedis = describe.skipIf(!HAS_REDIS); +const runWithVersionedBinding = describe.skipIf( + !HAS_REDIS || EXPECTED_CAPABILITY === "unavailable" +); +const TEST_PREFIX = `it-session-binding-${Date.now()}-${randomUUID()}`; +const BINDING_TTL_SECONDS = 90; +const COOLDOWN_TTL_SECONDS = 45; + +let redis: Redis; +let sequence = 0; +const touchedKeys = new Set(); + +function nextSessionId(label: string): string { + sequence += 1; + return `${TEST_PREFIX}:${label}:${sequence}`; +} + +function rememberBindingKeys( + sessionId: string, + keyIds: number[], + cooldownProviders: number[] = [] +): void { + for (const keyId of keyIds) { + const keys = buildSessionBindingKeys(sessionId, keyId); + touchedKeys.add(keys.canonical); + touchedKeys.add(keys.legacyProvider); + touchedKeys.add(keys.legacyOwner); + for (const providerId of cooldownProviders) { + touchedKeys.add(buildSessionProviderCooldownKey(sessionId, keyId, providerId)); + } + } +} + +async function deleteKeysIndividually(keys: Iterable): Promise { + for (const key of keys) { + await redis.del(key); + } +} + +async function scanKeys(pattern: string): Promise { + let cursor = "0"; + const keys: string[] = []; + do { + const [nextCursor, page] = await redis.scan(cursor, "MATCH", pattern, "COUNT", 100); + cursor = nextCursor; + keys.push(...page); + } while (cursor !== "0"); + return keys.sort(); +} + +async function cleanupTouchedKeys(): Promise { + await deleteKeysIndividually(touchedKeys); + touchedKeys.clear(); + + // Probe keys use a reserved isolated namespace and may remain only when a + // cluster rejects the probe's multi-key cleanup with CROSSSLOT. + const probeKeys = await scanKeys("session-binding-capability-probe:*"); + await deleteKeysIndividually(probeKeys); +} + +function requireOk(result: SessionBindingResult): SessionBindingOkResult { + if (result.status !== "ok") { + throw new Error(`Expected successful session binding result, got ${JSON.stringify(result)}`); + } + return result; +} + +async function readBinding(sessionId: string, keyId: number, ttlSeconds = BINDING_TTL_SECONDS) { + rememberBindingKeys(sessionId, [keyId]); + return readOrReconcileSessionBinding({ sessionId, keyId, ttlSeconds, redis }); +} + +async function bindProvider( + sessionId: string, + keyId: number, + expectedGeneration: string, + providerId: number +) { + rememberBindingKeys(sessionId, [keyId]); + return compareAndSetSessionBinding({ + sessionId, + keyId, + expectedGeneration, + providerId, + ttlSeconds: BINDING_TTL_SECONDS, + redis, + }); +} + +beforeAll(async () => { + if (!HAS_REDIS) return; + redis = new Redis(process.env.REDIS_URL!, { + lazyConnect: true, + enableOfflineQueue: false, + maxRetriesPerRequest: 1, + }); + await redis.connect(); + await expect(redis.ping()).resolves.toBe("PONG"); +}); + +beforeEach(() => { + resetVersionedBindingCapabilityForTests(); +}); + +afterEach(async () => { + if (HAS_REDIS) { + await cleanupTouchedKeys(); + } + resetVersionedBindingCapabilityForTests(); +}); + +afterAll(async () => { + if (HAS_REDIS) { + await cleanupTouchedKeys(); + if (redis.status !== "end") { + await redis.quit(); + } + } + resetVersionedBindingCapabilityForTests(); +}); + +runWithRedis("versioned session binding Redis capability", () => { + test("matches the explicitly expected capability and cleans isolated probe keys", async () => { + const probeKeysBefore = await scanKeys("session-binding-capability-probe:*"); + + const capability = await ensureVersionedBindingCapability(redis); + + expect(capability).toBe(EXPECTED_CAPABILITY); + if (capability === "available") { + expect(await scanKeys("session-binding-capability-probe:*")).toEqual(probeKeysBefore); + } + }); + + test.runIf(EXPECTED_CAPABILITY === "unavailable")( + "fails closed without creating a business binding when capability is unavailable", + async () => { + const sessionId = nextSessionId("unavailable"); + const keyId = 9001; + rememberBindingKeys(sessionId, [keyId]); + + const result = await readBinding(sessionId, keyId); + + expect(result).toMatchObject({ + status: "unavailable", + capabilityState: "unavailable", + legacyFallbackAllowed: true, + }); + expect(await redis.exists(buildCanonicalSessionBindingKey(sessionId, keyId))).toBe(0); + expect(await redis.exists(buildLegacySessionOwnerKey(sessionId))).toBe(0); + expect(await redis.exists(buildLegacySessionProviderKey(sessionId))).toBe(0); + } + ); +}); + +runWithVersionedBinding("versioned session binding reconcile", () => { + test("creates a true empty null tombstone and refreshes TTL without rotating generation", async () => { + const sessionId = nextSessionId("empty"); + const keyId = 1001; + const keys = buildSessionBindingKeys(sessionId, keyId); + + const created = requireOk(await readBinding(sessionId, keyId)); + expect(created).toMatchObject({ + source: "created", + snapshot: { sessionId, keyId, providerId: null }, + }); + expect(await redis.hget(keys.canonical, "key_id")).toBe(String(keyId)); + expect(await redis.hget(keys.canonical, "generation")).toBe(created.snapshot.generation); + expect(await redis.hget(keys.canonical, "provider_id")).toBeNull(); + expect(await redis.get(keys.legacyOwner)).toBe(String(keyId)); + expect(await redis.exists(keys.legacyProvider)).toBe(0); + + await redis.expire(keys.canonical, 5); + await redis.expire(keys.legacyOwner, 5); + + const reread = requireOk(await readBinding(sessionId, keyId)); + expect(reread.source).toBe("existing"); + expect(reread.snapshot.generation).toBe(created.snapshot.generation); + expect(reread.snapshot.providerId).toBeNull(); + expect(await redis.ttl(keys.canonical)).toBeGreaterThan(BINDING_TTL_SECONDS - 5); + expect(await redis.ttl(keys.legacyOwner)).toBeGreaterThan(BINDING_TTL_SECONDS - 5); + }); + + test("lazy-upgrades a matching legacy provider and owner", async () => { + const sessionId = nextSessionId("legacy-provider"); + const keyId = 1002; + const providerId = 2002; + const keys = buildSessionBindingKeys(sessionId, keyId); + rememberBindingKeys(sessionId, [keyId]); + await redis.setex(keys.legacyOwner, BINDING_TTL_SECONDS, String(keyId)); + await redis.setex(keys.legacyProvider, BINDING_TTL_SECONDS, String(providerId)); + + const upgraded = requireOk(await readBinding(sessionId, keyId)); + + expect(upgraded).toMatchObject({ + source: "legacy_upgraded", + snapshot: { sessionId, keyId, providerId }, + }); + expect(await redis.hgetall(keys.canonical)).toMatchObject({ + key_id: String(keyId), + generation: upgraded.snapshot.generation, + provider_id: String(providerId), + }); + }); + + test("lazy-upgrades a matching owner with no legacy provider as a null tombstone", async () => { + const sessionId = nextSessionId("legacy-null"); + const keyId = 1003; + const keys = buildSessionBindingKeys(sessionId, keyId); + rememberBindingKeys(sessionId, [keyId]); + await redis.setex(keys.legacyOwner, BINDING_TTL_SECONDS, String(keyId)); + + const upgraded = requireOk(await readBinding(sessionId, keyId)); + + expect(upgraded.source).toBe("legacy_upgraded"); + expect(upgraded.snapshot.providerId).toBeNull(); + expect(await redis.hget(keys.canonical, "provider_id")).toBeNull(); + expect(await redis.exists(keys.legacyProvider)).toBe(0); + }); + + test("rejects a foreign legacy owner without importing or overwriting it", async () => { + const sessionId = nextSessionId("foreign-owner"); + const keyId = 1004; + const foreignKeyId = 7777; + const providerId = 2004; + const keys = buildSessionBindingKeys(sessionId, keyId); + rememberBindingKeys(sessionId, [keyId]); + await redis.setex(keys.legacyOwner, BINDING_TTL_SECONDS, String(foreignKeyId)); + await redis.setex(keys.legacyProvider, BINDING_TTL_SECONDS, String(providerId)); + + const result = await readBinding(sessionId, keyId); + + expect(result).toEqual({ + status: "conflict", + reason: "foreign_legacy_owner", + legacyFallbackAllowed: false, + }); + expect(await redis.exists(keys.canonical)).toBe(0); + expect(await redis.get(keys.legacyOwner)).toBe(String(foreignKeyId)); + expect(await redis.get(keys.legacyProvider)).toBe(String(providerId)); + }); + + test("rejects an orphan legacy provider without claiming ownership", async () => { + const sessionId = nextSessionId("orphan-provider"); + const keyId = 1005; + const providerId = 2005; + const keys = buildSessionBindingKeys(sessionId, keyId); + rememberBindingKeys(sessionId, [keyId]); + await redis.setex(keys.legacyProvider, BINDING_TTL_SECONDS, String(providerId)); + + const result = await readBinding(sessionId, keyId); + + expect(result).toEqual({ + status: "conflict", + reason: "orphan_legacy_provider", + legacyFallbackAllowed: false, + }); + expect(await redis.exists(keys.canonical)).toBe(0); + expect(await redis.exists(keys.legacyOwner)).toBe(0); + expect(await redis.get(keys.legacyProvider)).toBe(String(providerId)); + }); + + test("rejects a non-positive legacy provider without creating canonical state", async () => { + const sessionId = nextSessionId("invalid-provider"); + const keyId = 1006; + const keys = buildSessionBindingKeys(sessionId, keyId); + rememberBindingKeys(sessionId, [keyId]); + await redis.setex(keys.legacyOwner, BINDING_TTL_SECONDS, String(keyId)); + await redis.setex(keys.legacyProvider, BINDING_TTL_SECONDS, "-1"); + + const result = await readBinding(sessionId, keyId); + + expect(result).toEqual({ + status: "conflict", + reason: "invalid_legacy_provider", + legacyFallbackAllowed: false, + }); + expect(await redis.exists(keys.canonical)).toBe(0); + }); + + test("fails closed for missing and contradictory mirrors without repairing either side", async () => { + const sessionId = nextSessionId("mirror-conflict"); + const keyId = 1006; + const providerId = 2006; + const keys = buildSessionBindingKeys(sessionId, keyId); + const created = requireOk(await readBinding(sessionId, keyId)); + const bound = requireOk( + await bindProvider(sessionId, keyId, created.snapshot.generation, providerId) + ); + + await redis.del(keys.legacyOwner); + const missing = await readBinding(sessionId, keyId); + expect(missing).toEqual({ + status: "conflict", + reason: "mirror_missing", + legacyFallbackAllowed: false, + }); + expect(await redis.hget(keys.canonical, "generation")).toBe(bound.snapshot.generation); + + await redis.setex(keys.legacyOwner, BINDING_TTL_SECONDS, String(keyId)); + await redis.setex(keys.legacyProvider, BINDING_TTL_SECONDS, String(providerId + 1)); + const contradictory = await readBinding(sessionId, keyId); + expect(contradictory).toEqual({ + status: "conflict", + reason: "mirror_conflict", + legacyFallbackAllowed: false, + }); + expect(await redis.hget(keys.canonical, "provider_id")).toBe(String(providerId)); + expect(await redis.get(keys.legacyProvider)).toBe(String(providerId + 1)); + }); + + test("allows only one tenant to initialize the same empty legacy session", async () => { + const sessionId = nextSessionId("tenant-race"); + const keyA = 1101; + const keyB = 1102; + rememberBindingKeys(sessionId, [keyA, keyB]); + + const [resultA, resultB] = await Promise.all([ + readOrReconcileSessionBinding({ + sessionId, + keyId: keyA, + ttlSeconds: BINDING_TTL_SECONDS, + redis, + }), + readOrReconcileSessionBinding({ + sessionId, + keyId: keyB, + ttlSeconds: BINDING_TTL_SECONDS, + redis, + }), + ]); + + const winner = resultA.status === "ok" ? resultA : resultB.status === "ok" ? resultB : null; + const loser = resultA.status === "conflict" ? resultA : resultB; + expect(winner).not.toBeNull(); + expect(loser).toMatchObject({ status: "conflict", reason: "foreign_legacy_owner" }); + if (!winner) throw new Error("Expected one tenant to win initialization"); + + const losingKeyId = winner.snapshot.keyId === keyA ? keyB : keyA; + expect(await redis.get(buildLegacySessionOwnerKey(sessionId))).toBe( + String(winner.snapshot.keyId) + ); + expect( + await redis.exists(buildCanonicalSessionBindingKey(sessionId, winner.snapshot.keyId)) + ).toBe(1); + expect(await redis.exists(buildCanonicalSessionBindingKey(sessionId, losingKeyId))).toBe(0); + }); +}); + +runWithVersionedBinding("versioned session binding mutation", () => { + test("touches exact null and provider snapshots without rotating generation", async () => { + const sessionId = nextSessionId("touch-exact"); + const keyId = 1200; + const providerId = 2200; + const keys = buildSessionBindingKeys(sessionId, keyId); + rememberBindingKeys(sessionId, [keyId]); + + const initial = requireOk(await readBinding(sessionId, keyId)); + await redis.expire(keys.canonical, 5); + await redis.expire(keys.legacyOwner, 5); + const touchedNull = requireOk( + await touchSessionBinding({ + ...initial.snapshot, + expectedGeneration: initial.snapshot.generation, + expectedProviderId: null, + ttlSeconds: BINDING_TTL_SECONDS, + redis, + }) + ); + expect(touchedNull.source).toBe("touched"); + expect(touchedNull.snapshot).toEqual(initial.snapshot); + expect(await redis.ttl(keys.canonical)).toBeGreaterThan(BINDING_TTL_SECONDS - 5); + expect(await redis.ttl(keys.legacyOwner)).toBeGreaterThan(BINDING_TTL_SECONDS - 5); + expect(await redis.exists(keys.legacyProvider)).toBe(0); + + const bound = requireOk( + await bindProvider(sessionId, keyId, touchedNull.snapshot.generation, providerId) + ); + await redis.expire(keys.canonical, 5); + await redis.expire(keys.legacyOwner, 5); + await redis.expire(keys.legacyProvider, 5); + const touchedProvider = requireOk( + await touchSessionBinding({ + sessionId, + keyId, + expectedGeneration: bound.snapshot.generation, + expectedProviderId: providerId, + ttlSeconds: BINDING_TTL_SECONDS, + redis, + }) + ); + expect(touchedProvider.source).toBe("touched"); + expect(touchedProvider.snapshot).toEqual(bound.snapshot); + expect(await redis.ttl(keys.canonical)).toBeGreaterThan(BINDING_TTL_SECONDS - 5); + expect(await redis.ttl(keys.legacyOwner)).toBeGreaterThan(BINDING_TTL_SECONDS - 5); + expect(await redis.ttl(keys.legacyProvider)).toBeGreaterThan(BINDING_TTL_SECONDS - 5); + }); + + test("rejects a stale touch after administrative termination advances generation", async () => { + const sessionId = nextSessionId("touch-after-admin-termination"); + const keyId = 1201; + const providerId = 2201; + const keys = buildSessionBindingKeys(sessionId, keyId); + rememberBindingKeys(sessionId, [keyId]); + + const initial = requireOk(await readBinding(sessionId, keyId)); + const bound = requireOk( + await bindProvider(sessionId, keyId, initial.snapshot.generation, providerId) + ); + const terminated = requireOk( + await terminateSessionBinding({ + sessionId, + keyId, + expectedProviderId: providerId, + ttlSeconds: BINDING_TTL_SECONDS, + redis, + }) + ); + + const staleTouch = await touchSessionBinding({ + sessionId, + keyId, + expectedGeneration: bound.snapshot.generation, + expectedProviderId: providerId, + ttlSeconds: BINDING_TTL_SECONDS, + redis, + }); + + expect(staleTouch).toEqual({ + status: "conflict", + reason: "generation_mismatch", + legacyFallbackAllowed: false, + }); + expect(await redis.hget(keys.canonical, "generation")).toBe(terminated.snapshot.generation); + expect(await redis.hget(keys.canonical, "provider_id")).toBeNull(); + expect(await redis.exists(keys.legacyProvider)).toBe(0); + }); + + test("rotates generation across CAS and rejects a stale ABA clear", async () => { + const sessionId = nextSessionId("aba"); + const keyId = 1201; + const providerP = 2201; + const providerQ = 2202; + rememberBindingKeys(sessionId, [keyId], [providerP]); + + const initial = requireOk(await readBinding(sessionId, keyId)); + const firstP = requireOk( + await bindProvider(sessionId, keyId, initial.snapshot.generation, providerP) + ); + + const staleCas = await bindProvider(sessionId, keyId, initial.snapshot.generation, providerQ); + expect(staleCas).toMatchObject({ status: "conflict", reason: "generation_mismatch" }); + + const boundQ = requireOk( + await bindProvider(sessionId, keyId, firstP.snapshot.generation, providerQ) + ); + const secondP = requireOk( + await bindProvider(sessionId, keyId, boundQ.snapshot.generation, providerP) + ); + expect( + new Set([ + initial.snapshot.generation, + firstP.snapshot.generation, + boundQ.snapshot.generation, + secondP.snapshot.generation, + ]).size + ).toBe(4); + + const staleClear = await clearSessionBinding({ + sessionId, + keyId, + expectedGeneration: firstP.snapshot.generation, + expectedProviderId: providerP, + ttlSeconds: BINDING_TTL_SECONDS, + cooldownTtlSeconds: COOLDOWN_TTL_SECONDS, + redis, + }); + expect(staleClear).toMatchObject({ status: "conflict", reason: "generation_mismatch" }); + expect(await redis.get(buildLegacySessionProviderKey(sessionId))).toBe(String(providerP)); + expect(await redis.hget(buildCanonicalSessionBindingKey(sessionId, keyId), "generation")).toBe( + secondP.snapshot.generation + ); + expect(await redis.exists(buildSessionProviderCooldownKey(sessionId, keyId, providerP))).toBe( + 0 + ); + }); + + test("atomically clears a provider and writes a tenant-scoped cooldown", async () => { + const sessionId = nextSessionId("clear-cooldown"); + const keyId = 1202; + const otherKeyId = 1203; + const providerId = 2203; + rememberBindingKeys(sessionId, [keyId, otherKeyId], [providerId]); + const keys = buildSessionBindingKeys(sessionId, keyId); + const cooldownKey = buildSessionProviderCooldownKey(sessionId, keyId, providerId); + + const initial = requireOk(await readBinding(sessionId, keyId)); + const bound = requireOk( + await bindProvider(sessionId, keyId, initial.snapshot.generation, providerId) + ); + const cleared = requireOk( + await clearSessionBinding({ + sessionId, + keyId, + expectedGeneration: bound.snapshot.generation, + expectedProviderId: providerId, + ttlSeconds: BINDING_TTL_SECONDS, + cooldownTtlSeconds: COOLDOWN_TTL_SECONDS, + redis, + }) + ); + + expect(cleared.source).toBe("cleared"); + expect(cleared.snapshot.providerId).toBeNull(); + expect(cleared.snapshot.generation).not.toBe(bound.snapshot.generation); + expect(await redis.hget(keys.canonical, "provider_id")).toBeNull(); + expect(await redis.get(keys.legacyOwner)).toBe(String(keyId)); + expect(await redis.exists(keys.legacyProvider)).toBe(0); + expect(await redis.get(cooldownKey)).toBe(cleared.snapshot.generation); + expect(await redis.ttl(cooldownKey)).toBeGreaterThan(COOLDOWN_TTL_SECONDS - 5); + + await expect( + isSessionProviderCoolingDown({ sessionId, keyId, providerId, redis }) + ).resolves.toEqual({ status: "ok", coolingDown: true, legacyFallbackAllowed: false }); + await expect( + isSessionProviderCoolingDown({ sessionId, keyId: otherKeyId, providerId, redis }) + ).resolves.toEqual({ status: "ok", coolingDown: false, legacyFallbackAllowed: false }); + }); + + test("does not initialize CAS state after canonical expiry", async () => { + const sessionId = nextSessionId("canonical-missing"); + const keyId = 1204; + const providerId = 2204; + const keys = buildSessionBindingKeys(sessionId, keyId); + const initial = requireOk(await readBinding(sessionId, keyId)); + await redis.del(keys.canonical); + + const result = await bindProvider(sessionId, keyId, initial.snapshot.generation, providerId); + + expect(result).toMatchObject({ status: "conflict", reason: "canonical_missing" }); + expect(await redis.exists(keys.canonical)).toBe(0); + expect(await redis.get(keys.legacyOwner)).toBe(String(keyId)); + expect(await redis.exists(keys.legacyProvider)).toBe(0); + }); + + test("fails closed when canonical generation is missing", async () => { + const sessionId = nextSessionId("generation-missing"); + const keyId = 1205; + const providerId = 2205; + const keys = buildSessionBindingKeys(sessionId, keyId); + const initial = requireOk(await readBinding(sessionId, keyId)); + await redis.hdel(keys.canonical, "generation"); + + const readResult = await readBinding(sessionId, keyId); + expect(readResult).toMatchObject({ status: "conflict", reason: "canonical_corrupt" }); + + const casResult = await bindProvider(sessionId, keyId, initial.snapshot.generation, providerId); + expect(casResult).toMatchObject({ status: "conflict", reason: "canonical_corrupt" }); + expect(await redis.exists(keys.legacyProvider)).toBe(0); + }); +}); diff --git a/tests/unit/lib/redis/client.test.ts b/tests/unit/lib/redis/client.test.ts index c6dfa6c49..4553739d5 100644 --- a/tests/unit/lib/redis/client.test.ts +++ b/tests/unit/lib/redis/client.test.ts @@ -116,6 +116,16 @@ describe("getRedisClient", () => { expect(mocks.MockRedis).toHaveBeenCalledTimes(1); }); + it("replaces the singleton when REDIS_URL changes", () => { + getRedisClient({ allowWhenRateLimitDisabled: true }); + process.env.REDIS_URL = "redis://localhost:6380"; + + getRedisClient({ allowWhenRateLimitDisabled: true }); + + expect(mocks.mockDisconnect).toHaveBeenCalledTimes(1); + expect(mocks.MockRedis).toHaveBeenCalledTimes(2); + }); + it("creates new client when existing singleton has status=end", () => { getRedisClient({ allowWhenRateLimitDisabled: true }); mocks.state.status = "end"; diff --git a/tests/unit/lib/redis/session-binding.test.ts b/tests/unit/lib/redis/session-binding.test.ts new file mode 100644 index 000000000..2831116b9 --- /dev/null +++ b/tests/unit/lib/redis/session-binding.test.ts @@ -0,0 +1,1604 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +vi.mock("server-only", () => ({})); +vi.mock("@/lib/logger", () => ({ + logger: { debug: vi.fn(), error: vi.fn(), info: vi.fn(), warn: vi.fn() }, +})); + +import { + acquireSessionDiscoveryLease, + buildCanonicalSessionBindingKey, + buildLegacySessionOwnerKey, + buildLegacySessionProviderKey, + buildSessionBindingKeys, + buildSessionDiscoveryLeaseKey, + buildSessionProviderCooldownKey, + clearSessionBinding, + compareAndSetSessionBinding, + ensureVersionedBindingCapability, + getVersionedBindingCapabilityState, + isSessionProviderCoolingDown, + mutateLegacySessionBindingSafely, + readOrReconcileSessionBinding, + refreshSessionBinding, + releaseSessionDiscoveryLease, + renewSessionDiscoveryLease, + resetVersionedBindingCapabilityForTests, + terminateSessionBinding, + touchSessionBinding, + type SessionBindingRedisClient, +} from "@/lib/redis/session-binding"; +import { + CAS_SESSION_BINDING, + CLEAR_SESSION_BINDING, + DELETE_LEGACY_PROVIDER_IF_VALUE, + READ_OR_RECONCILE_SESSION_BINDING, + RELEASE_SESSION_DISCOVERY_LEASE, + RENEW_SESSION_DISCOVERY_LEASE, + RESTORE_LEGACY_PROVIDER_IF_ABSENT, + TERMINATE_SESSION_BINDING, + TOUCH_SESSION_BINDING, +} from "@/lib/redis/lua-scripts"; + +type EvalResponse = unknown | Error | ((args: unknown[]) => unknown | Promise); + +interface MockRedisOptions { + cleanupFails?: boolean; + evalSha?: boolean; + evalShaNoScriptOnce?: boolean; + operationResponses?: Partial>; + probeFails?: boolean; + probeGate?: Promise; + status?: string; + cooldownValue?: string | null; + leaseSetResult?: unknown; +} + +function createMockRedis(options: MockRedisOptions = {}) { + const listeners = new Map void>>(); + const probeCooldowns = new Map(); + let probeFails = options.probeFails ?? false; + let cleanupFails = options.cleanupFails ?? false; + let status = options.status ?? "ready"; + + const evalMock = vi.fn(async (...args: unknown[]) => { + const [script, numberOfKeys] = args as [string, number]; + const firstKey = String(args[2]); + const isProbe = firstKey.startsWith("session-binding-capability-probe:"); + + if (isProbe) { + await options.probeGate; + if (probeFails) throw new Error("CROSSSLOT keys in request do not hash to the same slot"); + if (script === READ_OR_RECONCILE_SESSION_BINDING) { + return ["ok", "created", String(args[6]), ""]; + } + if (script === CAS_SESSION_BINDING) { + return ["ok", "updated", String(args[7]), String(args[8])]; + } + if (script === TOUCH_SESSION_BINDING) { + return ["ok", "touched", String(args[6]), String(args[7])]; + } + if (script === CLEAR_SESSION_BINDING) { + probeCooldowns.set(String(args[5]), String(args[8])); + return ["ok", "cleared", String(args[8]), ""]; + } + if (script === RENEW_SESSION_DISCOVERY_LEASE) return 1; + if (script === RELEASE_SESSION_DISCOVERY_LEASE) return 1; + throw new Error("Unexpected probe script"); + } + + const queue = options.operationResponses?.[script]; + const response = queue?.shift(); + if (response instanceof Error) throw response; + if (typeof response === "function") return response(args); + if (response !== undefined) return response; + if (script === DELETE_LEGACY_PROVIDER_IF_VALUE) { + const providerKey = String(args[2]); + const expectedProvider = String(args[3]); + if ((await getMock(providerKey)) !== expectedProvider) return 0; + await delMock(providerKey); + return 1; + } + throw new Error(`Missing operation response for ${numberOfKeys} key script`); + }); + + const getMock = vi.fn(async (key: string) => { + if (probeCooldowns.has(key)) return probeCooldowns.get(key) ?? null; + return options.cooldownValue ?? null; + }); + const hgetMock = vi.fn(async (_key: string, _field: string) => null as string | null); + const delMock = vi.fn(async (..._keys: string[]) => { + if (cleanupFails) throw new Error("cleanup failed"); + return 4; + }); + const existsMock = vi.fn(async () => 0); + const expireMock = vi.fn(async () => 1); + const setMock = vi.fn(async (key: string) => { + if (key.startsWith("session-binding-capability-probe:")) return "OK"; + return options.leaseSetResult === undefined ? "OK" : options.leaseSetResult; + }); + const setexMock = vi.fn(async () => "OK"); + const evalShaMock = vi.fn(async (..._args: unknown[]) => { + if (options.evalShaNoScriptOnce) { + options.evalShaNoScriptOnce = false; + throw new Error("NOSCRIPT No matching script"); + } + return ["ok", "existing", "generation-sha", "8"]; + }); + const onMock = vi.fn((event: string, listener: (...args: unknown[]) => void) => { + const callbacks = listeners.get(event) ?? new Set(); + callbacks.add(listener); + listeners.set(event, callbacks); + return redis; + }); + const offMock = vi.fn((event: string, listener: (...args: unknown[]) => void) => { + listeners.get(event)?.delete(listener); + return redis; + }); + + const redis = { + get status() { + return status; + }, + eval: evalMock, + ...(options.evalSha ? { evalsha: evalShaMock } : {}), + get: getMock, + hget: hgetMock, + del: delMock, + exists: existsMock, + expire: expireMock, + on: onMock, + off: offMock, + set: setMock, + setex: setexMock, + } satisfies SessionBindingRedisClient; + + return { + redis, + evalMock, + evalShaMock, + getMock, + hgetMock, + delMock, + existsMock, + expireMock, + setMock, + setexMock, + onMock, + offMock, + emit(event: string) { + for (const listener of listeners.get(event) ?? []) listener(); + }, + setStatus(nextStatus: string) { + status = nextStatus; + }, + setProbeFails(value: boolean) { + probeFails = value; + }, + setCleanupFails(value: boolean) { + cleanupFails = value; + }, + }; +} + +describe("session binding key builders", () => { + it("scopes canonical and cooldown keys by API key while preserving legacy mirrors", () => { + const canonical = buildCanonicalSessionBindingKey("session:{a}", 17); + const cooldown = buildSessionProviderCooldownKey("session:{a}", 17, 4); + const lease = buildSessionDiscoveryLeaseKey("session:{a}", 17); + + expect(canonical).toMatch(/^session-binding:v1:\{[a-f0-9]{64}\}:binding$/); + expect(cooldown).toMatch(/^session-binding:v1:\{[a-f0-9]{64}\}:provider:4:cooldown$/); + expect(lease).toMatch(/^session-binding:v1:\{[a-f0-9]{64}\}:discovery-lease$/); + expect(canonical.match(/\{([^}]+)\}/)?.[1]).toBe(cooldown.match(/\{([^}]+)\}/)?.[1]); + expect(canonical.match(/\{([^}]+)\}/)?.[1]).toBe(lease.match(/\{([^}]+)\}/)?.[1]); + expect(buildCanonicalSessionBindingKey("session:a", 18)).not.toBe( + buildCanonicalSessionBindingKey("session:a", 17) + ); + expect(buildSessionDiscoveryLeaseKey("session:a", 18)).not.toBe( + buildSessionDiscoveryLeaseKey("session:a", 17) + ); + expect(buildLegacySessionProviderKey("session:{a}")).toBe("session:session:{a}:provider"); + expect(buildLegacySessionOwnerKey("session:{a}")).toBe("session:session:{a}:key"); + }); + + it("supports an isolated namespace without changing the production key shape", () => { + const keys = buildSessionBindingKeys("sid", 9, "probe"); + expect(keys.canonical).toMatch(/^probe:session-binding:v1:\{[a-f0-9]{64}\}:binding$/); + expect(keys.legacyProvider).toBe("probe:session:sid:provider"); + expect(keys.legacyOwner).toBe("probe:session:sid:key"); + }); +}); + +describe("versioned binding capability", () => { + beforeEach(() => { + resetVersionedBindingCapabilityForTests(); + }); + + it("probes reconcile, CAS, touch, clear, cooldown, and cleanup exactly once per connection", async () => { + const mock = createMockRedis(); + + await expect(ensureVersionedBindingCapability(mock.redis)).resolves.toBe("available"); + await expect(ensureVersionedBindingCapability(mock.redis)).resolves.toBe("available"); + + expect(getVersionedBindingCapabilityState()).toBe("available"); + expect(mock.evalMock).toHaveBeenCalledTimes(6); + expect(mock.getMock).toHaveBeenCalledTimes(1); + expect(mock.setMock).toHaveBeenCalledTimes(1); + expect(mock.delMock).toHaveBeenCalledTimes(5); + expect(mock.delMock.mock.calls.every((call) => call.length === 1)).toBe(true); + expect(mock.onMock.mock.calls.map(([event]) => event)).toEqual([ + "close", + "connect", + "end", + "ready", + "reconnecting", + ]); + }); + + it("shares one in-flight capability probe across concurrent callers", async () => { + let releaseProbe: (() => void) | undefined; + const probeGate = new Promise((resolve) => { + releaseProbe = resolve; + }); + const mock = createMockRedis({ probeGate }); + + const first = ensureVersionedBindingCapability(mock.redis); + const second = ensureVersionedBindingCapability(mock.redis); + await vi.waitFor(() => expect(mock.evalMock).toHaveBeenCalledTimes(1)); + releaseProbe?.(); + + await expect(Promise.all([first, second])).resolves.toEqual(["available", "available"]); + expect(mock.evalMock).toHaveBeenCalledTimes(6); + }); + + it("stays unavailable on the same connection after a capability failure", async () => { + const mock = createMockRedis({ probeFails: true }); + + await expect(ensureVersionedBindingCapability(mock.redis)).resolves.toBe("unavailable"); + mock.setProbeFails(false); + await expect(ensureVersionedBindingCapability(mock.redis)).resolves.toBe("unavailable"); + + expect(mock.evalMock).toHaveBeenCalledTimes(1); + expect(getVersionedBindingCapabilityState()).toBe("unavailable"); + }); + + it("resets to unknown on reconnect and probes the new connection epoch", async () => { + const mock = createMockRedis({ probeFails: true }); + await ensureVersionedBindingCapability(mock.redis); + mock.setProbeFails(false); + + mock.emit("reconnecting"); + expect(getVersionedBindingCapabilityState()).toBe("unknown"); + await expect(ensureVersionedBindingCapability(mock.redis)).resolves.toBe("available"); + + expect(mock.evalMock).toHaveBeenCalledTimes(7); + }); + + it("automatically probes when a reconnected client becomes ready", async () => { + const mock = createMockRedis({ probeFails: true }); + await ensureVersionedBindingCapability(mock.redis); + mock.setProbeFails(false); + + mock.emit("close"); + expect(getVersionedBindingCapabilityState()).toBe("unknown"); + mock.emit("ready"); + + await vi.waitFor(() => expect(getVersionedBindingCapabilityState()).toBe("available")); + expect(mock.evalMock).toHaveBeenCalledTimes(7); + }); + + it("does not become available when isolated probe cleanup fails", async () => { + const mock = createMockRedis({ cleanupFails: true }); + + await expect(ensureVersionedBindingCapability(mock.redis)).resolves.toBe("unavailable"); + expect(mock.evalMock).toHaveBeenCalledTimes(6); + expect(mock.delMock).toHaveBeenCalledTimes(5); + }); + + it("detaches lifecycle listeners when tests reset state", async () => { + const mock = createMockRedis(); + await ensureVersionedBindingCapability(mock.redis); + + resetVersionedBindingCapabilityForTests(); + + expect(mock.offMock.mock.calls.map(([event]) => event)).toEqual([ + "close", + "connect", + "end", + "ready", + "reconnecting", + ]); + expect(getVersionedBindingCapabilityState()).toBe("unknown"); + }); + + it("reports unknown without creating a probe when Redis is not configured", async () => { + await expect(ensureVersionedBindingCapability()).resolves.toBe("unknown"); + expect(getVersionedBindingCapabilityState()).toBe("unknown"); + }); +}); + +describe("session Discovery lease operations", () => { + beforeEach(() => { + resetVersionedBindingCapabilityForTests(); + }); + + it("acquires a tenant-scoped lease with an explicit owner token and TTL", async () => { + const mock = createMockRedis(); + + const result = await acquireSessionDiscoveryLease({ + sessionId: "sid", + keyId: 4, + ttlSeconds: 61, + ownerToken: "owner-a", + redis: mock.redis, + }); + + expect(result).toEqual({ + status: "acquired", + ownerToken: "owner-a", + legacyFallbackAllowed: false, + }); + expect(mock.setMock.mock.calls.at(-1)).toEqual([ + buildSessionDiscoveryLeaseKey("sid", 4), + "owner-a", + "EX", + 61, + "NX", + ]); + }); + + it("returns a lease conflict without revealing the current owner", async () => { + const mock = createMockRedis({ leaseSetResult: null }); + + const result = await acquireSessionDiscoveryLease({ + sessionId: "sid", + keyId: 4, + ttlSeconds: 30, + redis: mock.redis, + }); + + expect(result).toEqual({ + status: "conflict", + reason: "lease_held", + legacyFallbackAllowed: false, + }); + }); + + it("renews and releases only through owner-token Lua primitives", async () => { + const mock = createMockRedis({ + operationResponses: { + [RENEW_SESSION_DISCOVERY_LEASE]: [1], + [RELEASE_SESSION_DISCOVERY_LEASE]: [1], + }, + }); + + await expect( + renewSessionDiscoveryLease({ + sessionId: "sid", + keyId: 4, + ownerToken: "owner-a", + ttlSeconds: 45, + redis: mock.redis, + }) + ).resolves.toEqual({ status: "renewed", legacyFallbackAllowed: false }); + await expect( + releaseSessionDiscoveryLease({ + sessionId: "sid", + keyId: 4, + ownerToken: "owner-a", + redis: mock.redis, + }) + ).resolves.toEqual({ status: "released", legacyFallbackAllowed: false }); + + expect(mock.evalMock).toHaveBeenCalledWith( + RENEW_SESSION_DISCOVERY_LEASE, + 1, + buildSessionDiscoveryLeaseKey("sid", 4), + "owner-a", + "45" + ); + expect(mock.evalMock).toHaveBeenCalledWith( + RELEASE_SESSION_DISCOVERY_LEASE, + 1, + buildSessionDiscoveryLeaseKey("sid", 4), + "owner-a" + ); + }); + + it("reports a lost lease when renew or release no longer owns the key", async () => { + const mock = createMockRedis({ + operationResponses: { + [RENEW_SESSION_DISCOVERY_LEASE]: [0], + [RELEASE_SESSION_DISCOVERY_LEASE]: [0], + }, + }); + + await expect( + renewSessionDiscoveryLease({ + sessionId: "sid", + keyId: 4, + ownerToken: "stale-owner", + ttlSeconds: 45, + redis: mock.redis, + }) + ).resolves.toEqual({ + status: "lost", + reason: "not_owner_or_missing", + legacyFallbackAllowed: false, + }); + await expect( + releaseSessionDiscoveryLease({ + sessionId: "sid", + keyId: 4, + ownerToken: "stale-owner", + redis: mock.redis, + }) + ).resolves.toEqual({ + status: "lost", + reason: "not_owner_or_missing", + legacyFallbackAllowed: false, + }); + }); + + it("rejects invalid identities before touching Redis", async () => { + const mock = createMockRedis(); + + await expect( + acquireSessionDiscoveryLease({ + sessionId: "", + keyId: 0, + ttlSeconds: 0, + ownerToken: "", + redis: mock.redis, + }) + ).resolves.toEqual({ + status: "conflict", + reason: "invalid_input", + legacyFallbackAllowed: false, + }); + await expect( + renewSessionDiscoveryLease({ + sessionId: "sid", + keyId: 4, + ttlSeconds: 30, + ownerToken: "", + redis: mock.redis, + }) + ).resolves.toEqual({ + status: "lost", + reason: "invalid_input", + legacyFallbackAllowed: false, + }); + + expect(mock.evalMock).not.toHaveBeenCalled(); + expect(mock.setMock).not.toHaveBeenCalled(); + }); +}); + +describe("versioned session binding operations", () => { + beforeEach(() => { + resetVersionedBindingCapabilityForTests(); + }); + + it("reads a newly initialized null tombstone", async () => { + const mock = createMockRedis({ + operationResponses: { + [READ_OR_RECONCILE_SESSION_BINDING]: [(args) => ["ok", "created", String(args[6]), ""]], + }, + }); + + const result = await readOrReconcileSessionBinding({ + sessionId: "fresh", + keyId: 7, + ttlSeconds: 90, + redis: mock.redis, + }); + + expect(result).toMatchObject({ + status: "ok", + source: "created", + snapshot: { sessionId: "fresh", keyId: 7, providerId: null }, + legacyFallbackAllowed: false, + }); + const operation = mock.evalMock.mock.calls.at(-1); + expect(operation?.slice(0, 5)).toEqual([ + READ_OR_RECONCILE_SESSION_BINDING, + 3, + buildCanonicalSessionBindingKey("fresh", 7), + "session:fresh:provider", + "session:fresh:key", + ]); + expect(operation?.at(-1)).toBe("90"); + }); + + it("parses an upgraded provider binding from Buffer values", async () => { + const mock = createMockRedis({ + operationResponses: { + [READ_OR_RECONCILE_SESSION_BINDING]: [ + [ + Buffer.from("ok"), + Buffer.from("legacy_upgraded"), + Buffer.from("generation-a"), + Buffer.from("12"), + ], + ], + }, + }); + + const result = await refreshSessionBinding({ + sessionId: "legacy", + keyId: 3, + redis: mock.redis, + }); + + expect(result).toEqual({ + status: "ok", + source: "legacy_upgraded", + snapshot: { + sessionId: "legacy", + keyId: 3, + providerId: 12, + generation: "generation-a", + }, + legacyFallbackAllowed: false, + }); + }); + + it("returns tenant conflicts without disabling capability", async () => { + const mock = createMockRedis({ + operationResponses: { + [READ_OR_RECONCILE_SESSION_BINDING]: [ + ["conflict", "foreign_legacy_owner"], + ["ok", "existing", "generation-b", "6"], + ], + }, + }); + + const first = await readOrReconcileSessionBinding({ + sessionId: "shared", + keyId: 2, + redis: mock.redis, + }); + const second = await readOrReconcileSessionBinding({ + sessionId: "shared", + keyId: 2, + redis: mock.redis, + }); + + expect(first).toEqual({ + status: "conflict", + reason: "foreign_legacy_owner", + legacyFallbackAllowed: false, + }); + expect(second.status).toBe("ok"); + expect(getVersionedBindingCapabilityState()).toBe("available"); + }); + + it("fails closed on unknown conflict reasons without disabling capability", async () => { + const mock = createMockRedis({ + operationResponses: { + [READ_OR_RECONCILE_SESSION_BINDING]: [["conflict", "future_conflict_reason"]], + }, + }); + + const result = await readOrReconcileSessionBinding({ + sessionId: "shared", + keyId: 2, + redis: mock.redis, + }); + + expect(result).toEqual({ + status: "conflict", + reason: "unknown_conflict", + legacyFallbackAllowed: false, + }); + expect(getVersionedBindingCapabilityState()).toBe("available"); + }); + + it("fails closed on malformed successful results without disabling capability", async () => { + const mock = createMockRedis({ + operationResponses: { + [READ_OR_RECONCILE_SESSION_BINDING]: [["ok", "existing"]], + }, + }); + + const result = await readOrReconcileSessionBinding({ + sessionId: "sid", + keyId: 2, + redis: mock.redis, + }); + + expect(result).toEqual({ + status: "conflict", + reason: "canonical_corrupt", + legacyFallbackAllowed: false, + }); + expect(getVersionedBindingCapabilityState()).toBe("available"); + }); + + it("CAS updates the provider and rotates generation", async () => { + const mock = createMockRedis({ + operationResponses: { + [CAS_SESSION_BINDING]: [(args) => ["ok", "updated", String(args[7]), String(args[8])]], + }, + }); + + const result = await compareAndSetSessionBinding({ + sessionId: "sid", + keyId: 4, + expectedGeneration: "old-generation", + providerId: 23, + redis: mock.redis, + }); + + expect(result.status).toBe("ok"); + if (result.status !== "ok") throw new Error("Expected successful CAS"); + expect(result.snapshot.providerId).toBe(23); + expect(result.snapshot.generation).not.toBe("old-generation"); + const operation = mock.evalMock.mock.calls.at(-1); + expect(operation?.[6]).toBe("old-generation"); + expect(operation?.[8]).toBe("23"); + }); + + it("returns generation conflicts without rotating global capability", async () => { + const mock = createMockRedis({ + operationResponses: { + [CAS_SESSION_BINDING]: [["conflict", "generation_mismatch"]], + }, + }); + + const result = await compareAndSetSessionBinding({ + sessionId: "sid", + keyId: 4, + expectedGeneration: "stale", + providerId: 23, + redis: mock.redis, + }); + + expect(result).toEqual({ + status: "conflict", + reason: "generation_mismatch", + legacyFallbackAllowed: false, + }); + expect(getVersionedBindingCapabilityState()).toBe("available"); + }); + + it.each([ + { label: "null tombstone", providerId: null }, + { label: "provider binding", providerId: 23 }, + ])("touches an exact $label without rotating generation", async ({ providerId }) => { + const mock = createMockRedis({ + operationResponses: { + [TOUCH_SESSION_BINDING]: [(args) => ["ok", "touched", String(args[6]), String(args[7])]], + }, + }); + + const result = await touchSessionBinding({ + sessionId: "sid", + keyId: 4, + expectedGeneration: "captured-generation", + expectedProviderId: providerId, + ttlSeconds: 90, + redis: mock.redis, + }); + + expect(result).toEqual({ + status: "ok", + source: "touched", + snapshot: { + sessionId: "sid", + keyId: 4, + providerId, + generation: "captured-generation", + }, + legacyFallbackAllowed: false, + }); + const operation = mock.evalMock.mock.calls.at(-1); + expect(operation?.slice(0, 5)).toEqual([ + TOUCH_SESSION_BINDING, + 3, + buildCanonicalSessionBindingKey("sid", 4), + "session:sid:provider", + "session:sid:key", + ]); + expect(operation?.slice(5)).toEqual([ + "4", + "captured-generation", + providerId?.toString() ?? "", + "90", + ]); + }); + + it.each([ + ["an advanced generation", "generation_mismatch"], + ["a different provider", "provider_mismatch"], + ["a missing canonical binding", "canonical_missing"], + ["a missing legacy mirror", "mirror_missing"], + ["a contradictory legacy mirror", "mirror_conflict"], + ] as const)("fails closed when touching %s", async (_label, reason) => { + const mock = createMockRedis({ + operationResponses: { + [TOUCH_SESSION_BINDING]: [["conflict", reason]], + }, + }); + + const result = await touchSessionBinding({ + sessionId: "sid", + keyId: 4, + expectedGeneration: "stale-or-conflicting-generation", + expectedProviderId: 23, + redis: mock.redis, + }); + + expect(result).toEqual({ + status: "conflict", + reason, + legacyFallbackAllowed: false, + }); + expect(getVersionedBindingCapabilityState()).toBe("available"); + }); + + it("clears a provider with a tenant-scoped cooldown in the same Lua call", async () => { + const mock = createMockRedis({ + cooldownValue: "cooldown-generation", + operationResponses: { + [CLEAR_SESSION_BINDING]: [(args) => ["ok", "cleared", String(args[8]), ""]], + }, + }); + + const result = await clearSessionBinding({ + sessionId: "sid", + keyId: 4, + expectedGeneration: "bound-generation", + expectedProviderId: 23, + cooldownTtlSeconds: 120, + redis: mock.redis, + }); + + expect(result.status).toBe("ok"); + if (result.status !== "ok") throw new Error("Expected successful clear"); + expect(result.snapshot.providerId).toBeNull(); + expect(result.snapshot.generation).not.toBe("bound-generation"); + const operation = mock.evalMock.mock.calls.at(-1); + expect(operation?.slice(2, 6)).toEqual([ + buildCanonicalSessionBindingKey("sid", 4), + "session:sid:provider", + "session:sid:key", + buildSessionProviderCooldownKey("sid", 4, 23), + ]); + expect(operation?.slice(-3)).toEqual(["300", "23", "120"]); + + const cooldown = await isSessionProviderCoolingDown({ + sessionId: "sid", + keyId: 4, + providerId: 23, + redis: mock.redis, + }); + expect(cooldown).toEqual({ + status: "ok", + coolingDown: true, + legacyFallbackAllowed: false, + }); + }); + + it("rotates a null tombstone without creating a cooldown key", async () => { + const mock = createMockRedis({ + operationResponses: { + [CLEAR_SESSION_BINDING]: [(args) => ["ok", "cleared", String(args[8]), ""]], + }, + }); + + const result = await clearSessionBinding({ + sessionId: "sid", + keyId: 4, + expectedGeneration: "null-generation", + expectedProviderId: null, + redis: mock.redis, + }); + + expect(result.status).toBe("ok"); + const operation = mock.evalMock.mock.calls.at(-1); + expect(operation?.[5]).toBe(buildCanonicalSessionBindingKey("sid", 4)); + expect(operation?.slice(-3)).toEqual(["300", "", "0"]); + }); + + it("rejects a cooldown without an expected provider before touching Redis", async () => { + const mock = createMockRedis(); + + const result = await clearSessionBinding({ + sessionId: "sid", + keyId: 4, + expectedGeneration: "generation", + expectedProviderId: null, + cooldownTtlSeconds: 30, + redis: mock.redis, + }); + + expect(result).toEqual({ + status: "conflict", + reason: "invalid_input", + legacyFallbackAllowed: false, + }); + expect(mock.evalMock).not.toHaveBeenCalled(); + }); + + it("marks operation errors unavailable and allows only infrastructure fallback", async () => { + const mock = createMockRedis({ + operationResponses: { + [READ_OR_RECONCILE_SESSION_BINDING]: [new Error("ERR script execution disabled")], + }, + }); + + const first = await readOrReconcileSessionBinding({ + sessionId: "sid", + keyId: 4, + redis: mock.redis, + }); + const callsAfterFailure = mock.evalMock.mock.calls.length; + const second = await readOrReconcileSessionBinding({ + sessionId: "sid", + keyId: 4, + redis: mock.redis, + }); + + expect(first).toMatchObject({ + status: "unavailable", + reason: "capability_unavailable", + capabilityState: "unavailable", + legacyFallbackAllowed: true, + }); + expect(second).toMatchObject({ + status: "unavailable", + reason: "capability_unavailable", + legacyFallbackAllowed: true, + }); + expect(mock.evalMock).toHaveBeenCalledTimes(callsAfterFailure); + }); + + it("allows legacy fallback on the first runtime capability error", async () => { + const mock = createMockRedis({ + operationResponses: { + [CAS_SESSION_BINDING]: [new Error("ERR script execution disabled")], + }, + }); + + const result = await compareAndSetSessionBinding({ + sessionId: "sid", + keyId: 4, + expectedGeneration: "generation-a", + providerId: 8, + redis: mock.redis, + }); + + expect(result).toMatchObject({ + status: "unavailable", + reason: "capability_unavailable", + capabilityState: "unavailable", + legacyFallbackAllowed: true, + }); + expect(getVersionedBindingCapabilityState()).toBe("unavailable"); + }); + + it("fails closed on malformed binding data without disabling the capability", async () => { + const mock = createMockRedis({ + operationResponses: { + [READ_OR_RECONCILE_SESSION_BINDING]: [ + ["ok", "existing", "generation-a", "invalid-provider"], + ["ok", "existing", "generation-b", "8"], + ], + }, + }); + + const first = await readOrReconcileSessionBinding({ + sessionId: "sid", + keyId: 4, + redis: mock.redis, + }); + const second = await readOrReconcileSessionBinding({ + sessionId: "sid", + keyId: 4, + redis: mock.redis, + }); + + expect(first).toEqual({ + status: "conflict", + reason: "canonical_corrupt", + legacyFallbackAllowed: false, + }); + expect(second.status).toBe("ok"); + expect(getVersionedBindingCapabilityState()).toBe("available"); + }); + + it("rejects a response from an obsolete connection epoch", async () => { + let resolveOperation: ((value: unknown) => void) | undefined; + const operation = new Promise((resolve) => { + resolveOperation = resolve; + }); + const mock = createMockRedis({ + operationResponses: { + [READ_OR_RECONCILE_SESSION_BINDING]: [() => operation], + }, + }); + + const pending = readOrReconcileSessionBinding({ + sessionId: "sid", + keyId: 4, + redis: mock.redis, + }); + await vi.waitFor(() => expect(mock.evalMock).toHaveBeenCalledTimes(7)); + mock.emit("reconnecting"); + resolveOperation?.(["ok", "existing", "generation", "8"]); + + await expect(pending).resolves.toMatchObject({ + status: "unavailable", + reason: "connection_changed", + legacyFallbackAllowed: false, + }); + expect(getVersionedBindingCapabilityState()).toBe("unknown"); + }); + + it("does not run Lua while Redis is not ready", async () => { + const mock = createMockRedis({ status: "connecting" }); + + const result = await readOrReconcileSessionBinding({ + sessionId: "sid", + keyId: 4, + redis: mock.redis, + }); + + expect(result).toMatchObject({ + status: "unavailable", + reason: "redis_not_ready", + legacyFallbackAllowed: true, + }); + expect(mock.evalMock).not.toHaveBeenCalled(); + }); + + it("treats invalid identities as non-fallback conflicts", async () => { + const mock = createMockRedis(); + + const result = await readOrReconcileSessionBinding({ + sessionId: "", + keyId: 0, + ttlSeconds: -1, + redis: mock.redis, + }); + + expect(result).toEqual({ + status: "conflict", + reason: "invalid_input", + legacyFallbackAllowed: false, + }); + expect(mock.evalMock).not.toHaveBeenCalled(); + }); + + it("returns false when a provider has no cooldown marker", async () => { + const mock = createMockRedis({ cooldownValue: null }); + + const result = await isSessionProviderCoolingDown({ + sessionId: "sid", + keyId: 4, + providerId: 9, + redis: mock.redis, + }); + + expect(result).toEqual({ + status: "ok", + coolingDown: false, + legacyFallbackAllowed: false, + }); + }); + + it("rejects foreign legacy state before any fallback mutation", async () => { + const mock = createMockRedis(); + mock.getMock.mockImplementation(async (key: string) => { + if (key === "session:sid:key") return "foreign-key"; + if (key === "session:sid:provider") return "9"; + return null; + }); + + const result = await mutateLegacySessionBindingSafely({ + sessionId: "sid", + keyId: 4, + redis: mock.redis, + mutation: { type: "set", providerId: 10 }, + }); + + expect(result).toEqual({ + status: "conflict", + reason: "foreign_legacy_owner", + legacyFallbackAllowed: false, + }); + expect(mock.setexMock).not.toHaveBeenCalled(); + }); + + it("claims a truly empty legacy owner with NX and rechecks it", async () => { + const mock = createMockRedis(); + let owner: string | null = null; + mock.getMock.mockImplementation(async (key: string) => { + if (key === "session:sid:key") return owner; + return null; + }); + mock.setMock.mockImplementation(async (_key: string, value: string) => { + owner = value; + return "OK"; + }); + + const result = await mutateLegacySessionBindingSafely({ + sessionId: "sid", + keyId: 4, + redis: mock.redis, + mutation: { type: "inspect" }, + }); + + expect(result).toEqual({ status: "ok", changed: false, providerId: null }); + expect(mock.setMock).toHaveBeenCalledWith("session:sid:key", "4", "EX", 300, "NX"); + }); + + it("blocks legacy mutation when canonical state already exists", async () => { + const mock = createMockRedis(); + mock.existsMock.mockResolvedValue(1); + + const result = await mutateLegacySessionBindingSafely({ + sessionId: "sid", + keyId: 4, + redis: mock.redis, + mutation: { type: "clear", expectedProviderId: 9 }, + }); + + expect(result).toEqual({ + status: "conflict", + reason: "canonical_exists", + legacyFallbackAllowed: false, + }); + expect(mock.getMock).not.toHaveBeenCalled(); + }); + + it("fails closed and rolls back its provider write if canonical state appears mid-mutation", async () => { + let provider: string | null = "8"; + const mock = createMockRedis({ + operationResponses: { + [DELETE_LEGACY_PROVIDER_IF_VALUE]: [ + () => { + provider = null; + return 1; + }, + ], + }, + }); + let existsCalls = 0; + mock.existsMock.mockImplementation(async () => { + existsCalls += 1; + // Initial guard and pre-mutation guard pass; the post-write guard sees + // a versioned worker creating the canonical binding concurrently. + return existsCalls === 3 ? 1 : 0; + }); + mock.getMock.mockImplementation(async (key: string) => { + if (key === "session:sid:key") return "4"; + if (key === "session:sid:provider") return provider; + return null; + }); + mock.setexMock.mockImplementation(async (key: string, _ttl: number, value: string) => { + if (key === "session:sid:provider") provider = value; + return "OK"; + }); + mock.delMock.mockImplementation(async (key: string) => { + if (key === "session:sid:provider") provider = null; + return 1; + }); + + const result = await mutateLegacySessionBindingSafely({ + sessionId: "sid", + keyId: 4, + redis: mock.redis, + mutation: { type: "set", providerId: 10 }, + }); + + expect(result).toEqual({ + status: "conflict", + reason: "canonical_exists", + legacyFallbackAllowed: false, + }); + expect(provider).toBeNull(); + expect(mock.evalMock).toHaveBeenCalledWith( + DELETE_LEGACY_PROVIDER_IF_VALUE, + 1, + "session:sid:provider", + "10" + ); + }); + + it("restores a cleared provider mirror if canonical state appears before the post-check", async () => { + let provider: string | null = "8"; + const mock = createMockRedis({ + operationResponses: { + [RESTORE_LEGACY_PROVIDER_IF_ABSENT]: [ + () => { + provider = "8"; + return 1; + }, + ], + }, + }); + let existsCalls = 0; + mock.existsMock.mockImplementation(async () => { + existsCalls += 1; + // Initial guard and pre-mutation guard pass; the post-clear guard sees + // a versioned worker creating canonical state concurrently. + return existsCalls === 3 ? 1 : 0; + }); + mock.getMock.mockImplementation(async (key: string) => { + if (key === "session:sid:key") return "4"; + if (key === "session:sid:provider") return provider; + return null; + }); + mock.hgetMock.mockResolvedValue("8"); + mock.delMock.mockImplementation(async (key: string) => { + if (key === "session:sid:provider") provider = null; + return 1; + }); + + const result = await mutateLegacySessionBindingSafely({ + sessionId: "sid", + keyId: 4, + redis: mock.redis, + mutation: { type: "clear", expectedProviderId: 8 }, + }); + + expect(result).toEqual({ + status: "conflict", + reason: "canonical_exists", + legacyFallbackAllowed: false, + }); + expect(provider).toBe("8"); + expect(mock.evalMock).toHaveBeenCalledWith( + RESTORE_LEGACY_PROVIDER_IF_ABSENT, + 1, + "session:sid:provider", + "8", + "300" + ); + }); + + it("keeps a provider mirror when canonical imported the same value before rollback", async () => { + let provider: string | null = null; + const mock = createMockRedis({ + operationResponses: { + [DELETE_LEGACY_PROVIDER_IF_VALUE]: [ + () => { + provider = null; + return 1; + }, + ], + }, + }); + let existsCalls = 0; + mock.existsMock.mockImplementation(async () => { + existsCalls += 1; + return existsCalls === 3 ? 1 : 0; + }); + mock.getMock.mockImplementation(async (key: string) => { + if (key === "session:sid:key") return "4"; + if (key === "session:sid:provider") return provider; + return null; + }); + mock.setexMock.mockImplementation(async (key: string, _ttl: number, value: string) => { + if (key === "session:sid:provider") provider = value; + return "OK"; + }); + mock.hgetMock.mockResolvedValue("10"); + + const result = await mutateLegacySessionBindingSafely({ + sessionId: "sid", + keyId: 4, + redis: mock.redis, + mutation: { type: "set", providerId: 10 }, + }); + + expect(result).toEqual({ + status: "conflict", + reason: "canonical_exists", + legacyFallbackAllowed: false, + }); + expect(provider).toBe("10"); + expect(mock.evalMock).not.toHaveBeenCalledWith( + DELETE_LEGACY_PROVIDER_IF_VALUE, + 1, + "session:sid:provider", + "10" + ); + }); + + it("rolls back a legacy provider bind when the owner cannot be refreshed", async () => { + let owner: string | null = "4"; + let provider: string | null = null; + const mock = createMockRedis({ + operationResponses: { + [DELETE_LEGACY_PROVIDER_IF_VALUE]: [ + () => { + provider = null; + return 1; + }, + ], + }, + }); + let expireCalls = 0; + mock.expireMock.mockImplementation(async () => { + expireCalls += 1; + if (expireCalls === 1) return 1; + owner = null; + return 0; + }); + mock.getMock.mockImplementation(async (key: string) => { + if (key === "session:sid:key") return owner; + if (key === "session:sid:provider") return provider; + return null; + }); + mock.setMock.mockImplementation(async () => { + owner = "5"; + return null; + }); + mock.setexMock.mockImplementation(async (key: string, _ttl: number, value: string) => { + if (key === "session:sid:provider") provider = value; + return "OK"; + }); + + const result = await mutateLegacySessionBindingSafely({ + sessionId: "sid", + keyId: 4, + redis: mock.redis, + mutation: { type: "set", providerId: 10 }, + }); + + expect(result).toEqual({ + status: "conflict", + reason: "foreign_legacy_owner", + legacyFallbackAllowed: false, + }); + expect(provider).toBeNull(); + expect(mock.evalMock).toHaveBeenCalledWith( + DELETE_LEGACY_PROVIDER_IF_VALUE, + 1, + "session:sid:provider", + "10" + ); + }); + + it("does not restore a mirror when the concurrent canonical binding is a null tombstone", async () => { + let provider: string | null = "8"; + const mock = createMockRedis(); + let existsCalls = 0; + mock.existsMock.mockImplementation(async () => { + existsCalls += 1; + return existsCalls === 3 ? 1 : 0; + }); + mock.getMock.mockImplementation(async (key: string) => { + if (key === "session:sid:key") return "4"; + if (key === "session:sid:provider") return provider; + return null; + }); + mock.hgetMock.mockResolvedValue(null); + mock.delMock.mockImplementation(async (key: string) => { + if (key === "session:sid:provider") provider = null; + return 1; + }); + + const result = await mutateLegacySessionBindingSafely({ + sessionId: "sid", + keyId: 4, + redis: mock.redis, + mutation: { type: "clear", expectedProviderId: 8 }, + }); + + expect(result).toMatchObject({ status: "conflict", reason: "canonical_exists" }); + expect(provider).toBeNull(); + expect(mock.evalMock).not.toHaveBeenCalledWith( + RESTORE_LEGACY_PROVIDER_IF_ABSENT, + expect.anything(), + expect.anything(), + expect.anything(), + expect.anything(), + expect.anything() + ); + }); + + it("does not clear a newer legacy Provider that replaced the expected value", async () => { + let provider: string | null = "8"; + const mock = createMockRedis({ + operationResponses: { + [DELETE_LEGACY_PROVIDER_IF_VALUE]: [ + () => { + provider = "9"; + return 0; + }, + ], + }, + }); + mock.getMock.mockImplementation(async (key: string) => { + if (key === "session:sid:key") return "4"; + if (key === "session:sid:provider") return provider; + return null; + }); + + const result = await mutateLegacySessionBindingSafely({ + sessionId: "sid", + keyId: 4, + redis: mock.redis, + mutation: { type: "clear", expectedProviderId: 8 }, + }); + + expect(result).toMatchObject({ status: "conflict", reason: "provider_mismatch" }); + expect(provider).toBe("9"); + expect(mock.delMock).not.toHaveBeenCalledWith("session:sid:provider"); + }); + + it("does not terminate a newer legacy Provider or its tenant owner", async () => { + let owner: string | null = "4"; + let provider: string | null = "8"; + const mock = createMockRedis({ + operationResponses: { + [DELETE_LEGACY_PROVIDER_IF_VALUE]: [ + () => { + provider = "9"; + return 0; + }, + ], + }, + }); + mock.getMock.mockImplementation(async (key: string) => { + if (key === "session:sid:key") return owner; + if (key === "session:sid:provider") return provider; + return null; + }); + mock.delMock.mockImplementation(async (key: string) => { + if (key === "session:sid:provider") provider = null; + if (key === "session:sid:key") owner = null; + return 1; + }); + + const result = await mutateLegacySessionBindingSafely({ + sessionId: "sid", + keyId: 4, + redis: mock.redis, + mutation: { type: "terminate", expectedProviderIds: [8] }, + }); + + expect(result).toMatchObject({ status: "conflict", reason: "provider_mismatch" }); + expect(provider).toBe("9"); + expect(owner).toBe("4"); + expect(mock.delMock).not.toHaveBeenCalled(); + }); + + it("keeps the tenant owner tombstone after an unscoped legacy termination", async () => { + let owner: string | null = "4"; + let provider: string | null = "8"; + const mock = createMockRedis(); + mock.getMock.mockImplementation(async (key: string) => { + if (key === "session:sid:key") return owner; + if (key === "session:sid:provider") return provider; + return null; + }); + mock.delMock.mockImplementation(async (key: string) => { + if (key === "session:sid:provider") provider = null; + if (key === "session:sid:key") owner = null; + return 1; + }); + + const result = await mutateLegacySessionBindingSafely({ + sessionId: "sid", + keyId: 4, + redis: mock.redis, + mutation: { type: "terminate" }, + }); + + expect(result).toMatchObject({ status: "ok", changed: true, providerId: null }); + expect(provider).toBeNull(); + expect(owner).toBe("4"); + expect(mock.delMock).not.toHaveBeenCalledWith("session:sid:key"); + }); + + it("restores mirrors when versioned recovery races an unscoped legacy termination", async () => { + let owner: string | null = "4"; + let provider: string | null = "8"; + const mock = createMockRedis({ + operationResponses: { + [RESTORE_LEGACY_PROVIDER_IF_ABSENT]: [ + () => { + if (provider === null) provider = "8"; + return 1; + }, + ], + }, + }); + let existsCalls = 0; + mock.existsMock.mockImplementation(async () => { + existsCalls += 1; + return existsCalls === 3 ? 1 : 0; + }); + mock.getMock.mockImplementation(async (key: string) => { + if (key === "session:sid:key") return owner; + if (key === "session:sid:provider") return provider; + return null; + }); + mock.hgetMock.mockResolvedValue("8"); + mock.delMock.mockImplementation(async (key: string) => { + if (key === "session:sid:provider") provider = null; + if (key === "session:sid:key") owner = null; + return 1; + }); + + const result = await mutateLegacySessionBindingSafely({ + sessionId: "sid", + keyId: 4, + redis: mock.redis, + mutation: { type: "terminate" }, + }); + + expect(result).toMatchObject({ status: "conflict", reason: "canonical_exists" }); + expect(owner).toBe("4"); + expect(provider).toBe("8"); + expect(mock.evalMock).toHaveBeenCalledWith( + DELETE_LEGACY_PROVIDER_IF_VALUE, + 1, + "session:sid:provider", + "8" + ); + expect(mock.evalMock).toHaveBeenCalledWith( + RESTORE_LEGACY_PROVIDER_IF_ABSENT, + 1, + "session:sid:provider", + "8", + "300" + ); + expect(mock.delMock).not.toHaveBeenCalledWith("session:sid:key"); + }); + + it("uses the tenant-authorized termination primitive and leaves a tombstone", async () => { + const mock = createMockRedis({ + operationResponses: { + [TERMINATE_SESSION_BINDING]: [(args) => ["ok", "terminated", String(args[6]), ""]], + }, + }); + + const result = await terminateSessionBinding({ + sessionId: "sid", + keyId: 4, + expectedProviderId: 9, + redis: mock.redis, + }); + + expect(result).toMatchObject({ + status: "ok", + source: "terminated", + snapshot: { sessionId: "sid", keyId: 4, providerId: null }, + }); + expect(mock.evalMock.mock.calls.at(-1)?.[8]).toBe("9"); + }); + + it("uses EVALSHA after capability warmup and falls back on NOSCRIPT", async () => { + const mock = createMockRedis({ + evalSha: true, + evalShaNoScriptOnce: true, + operationResponses: { + [READ_OR_RECONCILE_SESSION_BINDING]: [ + ["ok", "existing", "generation-a", "8"], + ["ok", "existing", "generation-b", "8"], + ], + }, + }); + + const first = await readOrReconcileSessionBinding({ + sessionId: "sid", + keyId: 4, + redis: mock.redis, + }); + const second = await readOrReconcileSessionBinding({ + sessionId: "sid", + keyId: 4, + redis: mock.redis, + }); + + expect(first.status).toBe("ok"); + expect(second.status).toBe("ok"); + expect(mock.evalShaMock).toHaveBeenCalled(); + expect(mock.evalMock).toHaveBeenCalled(); + }); + + it("covers tenant-safe legacy refresh, bind, set, clear, and terminate mutations", async () => { + const mock = createMockRedis(); + let owner: string | null = "4"; + let provider: string | null = null; + mock.getMock.mockImplementation(async (key: string) => { + if (key === "session:sid:key") return owner; + if (key === "session:sid:provider") return provider; + return null; + }); + mock.setMock.mockImplementation(async (key: string, value: string) => { + if (key === "session:sid:key") owner = value; + if (key === "session:sid:provider") provider = value; + return "OK"; + }); + mock.setexMock.mockImplementation(async (key: string, _ttl: number, value: string) => { + if (key === "session:sid:provider") provider = value; + if (key === "session:sid:key") owner = value; + return "OK"; + }); + mock.delMock.mockImplementation(async (key: string) => { + if (key === "session:sid:provider") provider = null; + if (key === "session:sid:key") owner = null; + return 1; + }); + + await expect( + mutateLegacySessionBindingSafely({ + sessionId: "sid", + keyId: 4, + redis: mock.redis, + mutation: { type: "refresh" }, + }) + ).resolves.toMatchObject({ status: "ok" }); + await expect( + mutateLegacySessionBindingSafely({ + sessionId: "sid", + keyId: 4, + redis: mock.redis, + mutation: { type: "bind_if_absent", providerId: 8 }, + }) + ).resolves.toMatchObject({ status: "ok", changed: true, providerId: 8 }); + await expect( + mutateLegacySessionBindingSafely({ + sessionId: "sid", + keyId: 4, + redis: mock.redis, + mutation: { type: "set", providerId: 9 }, + }) + ).resolves.toMatchObject({ status: "ok", changed: true, providerId: 9 }); + await expect( + mutateLegacySessionBindingSafely({ + sessionId: "sid", + keyId: 4, + redis: mock.redis, + mutation: { type: "clear", expectedProviderId: 9 }, + }) + ).resolves.toMatchObject({ status: "ok", changed: true, providerId: null }); + + provider = "10"; + await expect( + mutateLegacySessionBindingSafely({ + sessionId: "sid", + keyId: 4, + redis: mock.redis, + mutation: { type: "terminate", expectedProviderIds: [10] }, + }) + ).resolves.toMatchObject({ + status: "ok", + changed: true, + providerId: null, + terminatedProviderId: 10, + }); + expect(owner).toBe("4"); + }); + + it("rejects malformed legacy mutation arguments before touching Redis", async () => { + const mock = createMockRedis(); + + await expect( + mutateLegacySessionBindingSafely({ + sessionId: "sid", + keyId: 4, + redis: mock.redis, + mutation: { type: "set", providerId: 0 }, + }) + ).resolves.toMatchObject({ status: "conflict", reason: "invalid_input" }); + await expect( + mutateLegacySessionBindingSafely({ + sessionId: "sid", + keyId: 4, + redis: mock.redis, + mutation: { type: "terminate", expectedProviderIds: [0] }, + }) + ).resolves.toMatchObject({ status: "conflict", reason: "invalid_input" }); + expect(mock.existsMock).not.toHaveBeenCalled(); + }); +}); diff --git a/tests/unit/lib/session-manager-binding-smart.test.ts b/tests/unit/lib/session-manager-binding-smart.test.ts index 8186ff9b8..62c188797 100644 --- a/tests/unit/lib/session-manager-binding-smart.test.ts +++ b/tests/unit/lib/session-manager-binding-smart.test.ts @@ -3,9 +3,9 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; /** * Tests for SessionManager.updateSessionBindingSmart forceUpdate semantics. * - * Hedge race winners must unconditionally rebind the session-reuse binding to - * the winner. forceUpdate short-circuits the smart-decision path (priority / - * circuit health) that would otherwise keep a healthy higher-priority binding. + * Hedge race winners bypass the smart-decision path (priority / circuit + * health), while versioned bindings still use generation CAS so a stale winner + * cannot overwrite a newer concurrent binding. */ let redisClientRef: { @@ -21,6 +21,15 @@ let lastPipeline: { exec: ReturnType; }; +const bindingMocks = vi.hoisted(() => ({ + clearSessionBinding: vi.fn(), + compareAndSetSessionBinding: vi.fn(), + isSessionProviderCoolingDown: vi.fn(), + mutateLegacySessionBindingSafely: vi.fn(), + readOrReconcileSessionBinding: vi.fn(), + refreshSessionBinding: vi.fn(), +})); + const makePipeline = () => { const pipeline = { setex: vi.fn(() => pipeline), @@ -44,6 +53,11 @@ vi.mock("@/lib/redis", () => ({ getRedisClient: () => redisClientRef, })); +vi.mock("@/lib/redis/session-binding", () => ({ + ...bindingMocks, + getVersionedBindingCapabilityState: () => "unavailable", +})); + // Both are loaded via `await import(...)` inside updateSessionBindingSmart; the // static vi.mock still intercepts the dynamic import. vi.mock("@/repository/provider", () => ({ @@ -59,10 +73,35 @@ import { SessionManager } from "@/lib/session-manager"; import { findProviderById } from "@/repository/provider"; const SID = "sess-binding"; -const TTL = 300; +const KEY_ID = 42; +let legacyProviderId: number | null; beforeEach(() => { vi.clearAllMocks(); + bindingMocks.readOrReconcileSessionBinding.mockResolvedValue({ + status: "unavailable", + reason: "capability_unavailable", + capabilityState: "unavailable", + legacyFallbackAllowed: true, + }); + legacyProviderId = null; + bindingMocks.mutateLegacySessionBindingSafely.mockImplementation(async (input: any) => { + if (input.mutation.type === "inspect") { + return { status: "ok", changed: false, providerId: legacyProviderId }; + } + if (input.mutation.type === "bind_if_absent") { + if (legacyProviderId !== null) { + return { status: "ok", changed: false, providerId: legacyProviderId }; + } + legacyProviderId = input.mutation.providerId; + return { status: "ok", changed: true, providerId: legacyProviderId }; + } + if (input.mutation.type === "set") { + legacyProviderId = input.mutation.providerId; + return { status: "ok", changed: true, providerId: legacyProviderId }; + } + throw new Error(`Unexpected mutation ${input.mutation.type}`); + }); redisClientRef = { status: "ready", get: vi.fn(async () => null), @@ -75,7 +114,7 @@ beforeEach(() => { describe("SessionManager.updateSessionBindingSmart forceUpdate", () => { it("forceUpdate=true overrides a healthy higher-priority existing binding", async () => { // Existing binding -> provider 1 (healthy, higher priority than the winner) - redisClientRef!.get.mockResolvedValue("1"); + legacyProviderId = 1; vi.mocked(findProviderById).mockResolvedValue({ id: 1, name: "main", priority: 5 } as never); vi.mocked(isCircuitOpen).mockResolvedValue(false); @@ -85,20 +124,20 @@ describe("SessionManager.updateSessionBindingSmart forceUpdate", () => { 10, // winner priority (lower priority than current's 5) false, // isFirstAttempt false, // isFailoverSuccess - null, + KEY_ID, true // forceUpdate ); expect(result).toMatchObject({ updated: true, reason: "race_winner_forced" }); - expect(lastPipeline.setex).toHaveBeenCalledWith(`session:${SID}:provider`, TTL, "2"); - // Guard against a regression that queues setex but forgets to flush the pipeline. - expect(lastPipeline.exec).toHaveBeenCalledTimes(1); + expect(bindingMocks.mutateLegacySessionBindingSafely).toHaveBeenCalledWith( + expect.objectContaining({ mutation: { type: "set", providerId: 2 } }) + ); }); it("forceUpdate=true rebinds even when the winner equals the current binding", async () => { // Production winner==initialProvider race: the bound provider is already the winner, // but the race result must still (re)write the binding and refresh its TTL. - redisClientRef!.get.mockResolvedValue("2"); + legacyProviderId = 2; const result = await SessionManager.updateSessionBindingSmart( SID, @@ -106,18 +145,18 @@ describe("SessionManager.updateSessionBindingSmart forceUpdate", () => { 10, false, false, - null, + KEY_ID, true // forceUpdate ); expect(result).toMatchObject({ updated: true, reason: "race_winner_forced" }); - expect(redisClientRef!.get).not.toHaveBeenCalled(); - expect(lastPipeline.setex).toHaveBeenCalledWith(`session:${SID}:provider`, TTL, "2"); - expect(lastPipeline.exec).toHaveBeenCalledTimes(1); + expect(bindingMocks.mutateLegacySessionBindingSafely).toHaveBeenCalledWith( + expect.objectContaining({ mutation: { type: "set", providerId: 2 } }) + ); }); it("forceUpdate=false keeps the healthy higher-priority binding (documents the gap)", async () => { - redisClientRef!.get.mockResolvedValue("1"); + legacyProviderId = 1; vi.mocked(findProviderById).mockResolvedValue({ id: 1, name: "main", priority: 5 } as never); vi.mocked(isCircuitOpen).mockResolvedValue(false); @@ -127,7 +166,7 @@ describe("SessionManager.updateSessionBindingSmart forceUpdate", () => { 10, false, false, - null, + KEY_ID, false // forceUpdate ); @@ -135,14 +174,14 @@ describe("SessionManager.updateSessionBindingSmart forceUpdate", () => { }); it("forceUpdate=true short-circuits before consulting provider/circuit state", async () => { - redisClientRef!.get.mockResolvedValue("1"); + legacyProviderId = 1; - await SessionManager.updateSessionBindingSmart(SID, 2, 10, false, false, null, true); + await SessionManager.updateSessionBindingSmart(SID, 2, 10, false, false, KEY_ID, true); expect(findProviderById).not.toHaveBeenCalled(); expect(isCircuitOpen).not.toHaveBeenCalled(); - // forceUpdate goes straight to the unconditional pipeline path. - expect(redisClientRef!.get).not.toHaveBeenCalled(); + // forceUpdate goes straight to the persistence path. + expect(findProviderById).not.toHaveBeenCalled(); }); it("forceUpdate=true also persists the keyId binding with TTL", async () => { @@ -152,14 +191,18 @@ describe("SessionManager.updateSessionBindingSmart forceUpdate", () => { 10, false, false, - 42, // keyId + KEY_ID, true ); expect(result.updated).toBe(true); - expect(lastPipeline.setex).toHaveBeenCalledWith(`session:${SID}:provider`, TTL, "2"); - expect(lastPipeline.setex).toHaveBeenCalledWith(`session:${SID}:key`, TTL, "42"); - expect(lastPipeline.exec).toHaveBeenCalledTimes(1); + expect(bindingMocks.mutateLegacySessionBindingSafely).toHaveBeenCalledWith( + expect.objectContaining({ + sessionId: SID, + keyId: KEY_ID, + mutation: { type: "set", providerId: 2 }, + }) + ); }); it("isFailoverSuccess=true keeps reason failover_success even when forceUpdate=true", async () => { @@ -169,7 +212,7 @@ describe("SessionManager.updateSessionBindingSmart forceUpdate", () => { 10, false, true, // isFailoverSuccess - null, + KEY_ID, true // forceUpdate ); @@ -191,4 +234,148 @@ describe("SessionManager.updateSessionBindingSmart forceUpdate", () => { expect(result).toMatchObject({ updated: false, reason: "redis_not_ready" }); }); + + it("uses generation CAS instead of legacy writes when versioned binding is available", async () => { + bindingMocks.readOrReconcileSessionBinding.mockResolvedValue({ + status: "ok", + source: "existing", + snapshot: { + sessionId: SID, + keyId: 42, + providerId: 1, + generation: "generation-a", + }, + legacyFallbackAllowed: false, + }); + bindingMocks.compareAndSetSessionBinding.mockResolvedValue({ + status: "ok", + source: "updated", + snapshot: { + sessionId: SID, + keyId: 42, + providerId: 2, + generation: "generation-b", + }, + legacyFallbackAllowed: false, + }); + + const result = await SessionManager.updateSessionBindingSmart( + SID, + 2, + 10, + false, + false, + 42, + true + ); + + expect(result).toMatchObject({ updated: true, reason: "race_winner_forced" }); + expect(bindingMocks.compareAndSetSessionBinding).toHaveBeenCalledWith( + expect.objectContaining({ + sessionId: SID, + keyId: 42, + expectedGeneration: "generation-a", + providerId: 2, + }) + ); + expect(result.bindingSnapshot).toEqual({ + sessionId: SID, + keyId: 42, + providerId: 2, + generation: "generation-b", + }); + expect(result.legacyBindingUpdated).toBeUndefined(); + expect(redisClientRef!.pipeline).not.toHaveBeenCalled(); + expect(redisClientRef!.setex).not.toHaveBeenCalled(); + }); + + it("marks only a confirmed legacy force-update as eligible for legacy cleanup", async () => { + legacyProviderId = 1; + + const result = await SessionManager.updateSessionBindingSmart( + SID, + 2, + 10, + false, + false, + KEY_ID, + true + ); + + expect(result).toMatchObject({ + updated: true, + reason: "race_winner_forced", + legacyBindingUpdated: true, + }); + expect(result.bindingSnapshot).toBeUndefined(); + }); + + it.each([ + { isFailoverSuccess: false, forceUpdate: true }, + { isFailoverSuccess: true, forceUpdate: false }, + ])( + "does not let a stale versioned winner overwrite a newer binding (%o)", + async ({ isFailoverSuccess, forceUpdate }) => { + bindingMocks.readOrReconcileSessionBinding.mockResolvedValue({ + status: "ok", + source: "existing", + snapshot: { + sessionId: SID, + keyId: KEY_ID, + providerId: 1, + generation: "generation-before-concurrent-update", + }, + legacyFallbackAllowed: false, + }); + bindingMocks.compareAndSetSessionBinding.mockResolvedValue({ + status: "conflict", + reason: "generation_mismatch", + legacyFallbackAllowed: false, + }); + + const result = await SessionManager.updateSessionBindingSmart( + SID, + 2, + 10, + false, + isFailoverSuccess, + KEY_ID, + forceUpdate + ); + + expect(result).toEqual({ + updated: false, + reason: "concurrent_binding_changed", + details: "Session binding changed before the update committed", + }); + expect(bindingMocks.compareAndSetSessionBinding).toHaveBeenCalledTimes(1); + expect(bindingMocks.mutateLegacySessionBindingSafely).not.toHaveBeenCalled(); + } + ); + + it("does not fall back to legacy writes for a foreign owner conflict", async () => { + bindingMocks.readOrReconcileSessionBinding.mockResolvedValue({ + status: "conflict", + reason: "foreign_legacy_owner", + legacyFallbackAllowed: false, + }); + + const result = await SessionManager.updateSessionBindingSmart( + SID, + 2, + 10, + false, + false, + 42, + true + ); + + expect(result).toEqual({ + updated: false, + reason: "versioned_binding_conflict", + details: "foreign_legacy_owner", + }); + expect(redisClientRef!.pipeline).not.toHaveBeenCalled(); + expect(redisClientRef!.set).not.toHaveBeenCalled(); + }); }); diff --git a/tests/unit/lib/session-manager-content-hash.test.ts b/tests/unit/lib/session-manager-content-hash.test.ts new file mode 100644 index 000000000..1cd2c393c --- /dev/null +++ b/tests/unit/lib/session-manager-content-hash.test.ts @@ -0,0 +1,198 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +let redisClientRef: { + status: string; + get: ReturnType; + setex: ReturnType; + pipeline: ReturnType; +}; +let values: Map; + +const bindingMocks = vi.hoisted(() => ({ + mutateLegacySessionBindingSafely: vi.fn(), + readOrReconcileSessionBinding: vi.fn(), +})); + +vi.mock("server-only", () => ({})); + +vi.mock("@/lib/logger", () => ({ + logger: { + debug: vi.fn(), + error: vi.fn(), + info: vi.fn(), + trace: vi.fn(), + warn: vi.fn(), + }, +})); + +vi.mock("@/lib/redis", () => ({ + getRedisClient: () => redisClientRef, +})); + +vi.mock("@/lib/redis/session-binding", () => ({ + ...bindingMocks, + getVersionedBindingCapabilityState: () => "available", +})); + +vi.mock("@/lib/session-tracker", () => ({ + SessionTracker: { + getConcurrentCount: vi.fn(async () => 0), + }, +})); + +import { SessionManager } from "@/lib/session-manager"; + +const MESSAGES = [{ role: "user", content: "identical tenant-sensitive prompt" }]; + +function contentHash(): string { + const hash = SessionManager.calculateMessagesHash(MESSAGES); + if (!hash) throw new Error("Expected test messages to produce a content hash"); + return hash; +} + +function tenantHashKey(keyId: number): string { + return `hash:${keyId}:${contentHash()}:session`; +} + +function legacyHashKey(): string { + return `hash:${contentHash()}:session`; +} + +beforeEach(() => { + vi.clearAllMocks(); + values = new Map(); + + const createPipeline = () => { + const operations: Array<() => void> = []; + const pipeline = { + setex: vi.fn((key: string, _ttlSeconds: number, value: string) => { + operations.push(() => values.set(key, value)); + return pipeline; + }), + exec: vi.fn(async () => { + for (const operation of operations) operation(); + return operations.map(() => [null, "OK"]); + }), + }; + return pipeline; + }; + + redisClientRef = { + status: "ready", + get: vi.fn(async (key: string) => values.get(key) ?? null), + setex: vi.fn(async (key: string, _ttlSeconds: number, value: string) => { + values.set(key, value); + return "OK"; + }), + pipeline: vi.fn(createPipeline), + }; + + bindingMocks.readOrReconcileSessionBinding.mockImplementation( + async ({ sessionId, keyId }: { sessionId: string; keyId: number }) => { + values.set(`session:${sessionId}:key`, keyId.toString()); + return { + status: "ok", + source: "created", + snapshot: { + sessionId, + keyId, + providerId: null, + generation: `generation-${keyId}`, + }, + legacyFallbackAllowed: false, + }; + } + ); +}); + +describe("SessionManager content-hash mapping tenant isolation", () => { + it("does not share a generated Session between API keys with identical content", async () => { + const first = await SessionManager.getOrCreateSessionId(101, MESSAGES, null); + await vi.waitFor(() => expect(values.get(tenantHashKey(101))).toBe(first)); + + const second = await SessionManager.getOrCreateSessionId(202, MESSAGES, null); + await vi.waitFor(() => expect(values.get(tenantHashKey(202))).toBe(second)); + + expect(second).not.toBe(first); + expect(values.has(legacyHashKey())).toBe(false); + }); + + it("continues to reuse the tenant-scoped mapping for the same API key", async () => { + const first = await SessionManager.getOrCreateSessionId(101, MESSAGES, null); + await vi.waitFor(() => expect(values.get(tenantHashKey(101))).toBe(first)); + + bindingMocks.readOrReconcileSessionBinding.mockClear(); + const second = await SessionManager.getOrCreateSessionId(101, MESSAGES, null); + + expect(second).toBe(first); + expect(bindingMocks.readOrReconcileSessionBinding).toHaveBeenCalledWith( + expect.objectContaining({ sessionId: first, keyId: 101 }) + ); + }); + + it.each([ + ["matching", "101"], + ["foreign", "202"], + ["missing", null], + ])("never reads or imports an unscoped legacy mapping with a %s owner", async (_case, owner) => { + const legacySessionId = `legacy-${_case}-owner-session`; + values.set(legacyHashKey(), legacySessionId); + if (owner !== null) values.set(`session:${legacySessionId}:key`, owner); + + const result = await SessionManager.getOrCreateSessionId(101, MESSAGES, null); + + expect(result).not.toBe(legacySessionId); + await vi.waitFor(() => expect(values.get(tenantHashKey(101))).toBe(result)); + expect(redisClientRef.get).not.toHaveBeenCalledWith(legacyHashKey()); + expect(values.get(legacyHashKey())).toBe(legacySessionId); + expect(values.get(`session:${legacySessionId}:key`)).toBe(owner ?? undefined); + }); + + it.each([ + ["missing", null], + ["foreign", "202"], + ["corrupt", "not-a-key-id"], + ])("rejects a tenant-scoped mapping with a %s owner", async (_case, owner) => { + const staleSessionId = `scoped-${_case}-owner-session`; + values.set(tenantHashKey(101), staleSessionId); + if (owner !== null) values.set(`session:${staleSessionId}:key`, owner); + + const result = await SessionManager.getOrCreateSessionId(101, MESSAGES, null); + + expect(result).not.toBe(staleSessionId); + await vi.waitFor(() => expect(values.get(tenantHashKey(101))).toBe(result)); + expect(values.get(`session:${staleSessionId}:key`)).toBe(owner ?? undefined); + }); + + it("rejects a tenant-scoped mapping when canonical reconciliation fails", async () => { + const staleSessionId = "scoped-conflicting-binding-session"; + values.set(tenantHashKey(101), staleSessionId); + values.set(`session:${staleSessionId}:key`, "101"); + bindingMocks.readOrReconcileSessionBinding.mockResolvedValueOnce({ + status: "conflict", + reason: "mirror_mismatch", + legacyFallbackAllowed: false, + }); + + const result = await SessionManager.getOrCreateSessionId(101, MESSAGES, null); + + expect(result).not.toBe(staleSessionId); + await vi.waitFor(() => expect(values.get(tenantHashKey(101))).toBe(result)); + }); + + it("does not publish a tenant-scoped mapping when binding initialization fails", async () => { + bindingMocks.readOrReconcileSessionBinding.mockResolvedValueOnce({ + status: "conflict", + reason: "foreign_owner", + legacyFallbackAllowed: false, + }); + + await SessionManager.getOrCreateSessionId(101, MESSAGES, null); + + await vi.waitFor(() => { + expect(bindingMocks.readOrReconcileSessionBinding).toHaveBeenCalledTimes(1); + }); + expect(values.has(tenantHashKey(101))).toBe(false); + expect(redisClientRef.pipeline).not.toHaveBeenCalled(); + }); +}); diff --git a/tests/unit/lib/session-manager-terminate-provider-sessions.test.ts b/tests/unit/lib/session-manager-terminate-provider-sessions.test.ts index 665a979c0..30d346484 100644 --- a/tests/unit/lib/session-manager-terminate-provider-sessions.test.ts +++ b/tests/unit/lib/session-manager-terminate-provider-sessions.test.ts @@ -50,6 +50,9 @@ describe("SessionManager.terminateProviderSessionsBatch", () => { expect(pipelineRef.zrange).toHaveBeenCalledTimes(2); expect(pipelineRef.zrange).toHaveBeenCalledWith("provider:42:active_sessions", 0, -1); expect(pipelineRef.zrange).toHaveBeenCalledWith("provider:43:active_sessions", 0, -1); - expect(terminateSessionsBatchSpy).toHaveBeenCalledWith(["sess-a", "sess-b", "sess-c"]); + expect(terminateSessionsBatchSpy).toHaveBeenCalledWith( + ["sess-a", "sess-b", "sess-c"], + [42, 43] + ); }); }); diff --git a/tests/unit/lib/session-manager-terminate-session.test.ts b/tests/unit/lib/session-manager-terminate-session.test.ts index 76145bd34..05f6ac1b2 100644 --- a/tests/unit/lib/session-manager-terminate-session.test.ts +++ b/tests/unit/lib/session-manager-terminate-session.test.ts @@ -2,6 +2,15 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; let redisClientRef: any; let pipelineRef: any; +const bindingMocks = vi.hoisted(() => ({ + clearSessionBinding: vi.fn(), + compareAndSetSessionBinding: vi.fn(), + isSessionProviderCoolingDown: vi.fn(), + mutateLegacySessionBindingSafely: vi.fn(), + readOrReconcileSessionBinding: vi.fn(), + refreshSessionBinding: vi.fn(), + terminateSessionBinding: vi.fn(), +})); vi.mock("server-only", () => ({})); @@ -19,6 +28,11 @@ vi.mock("@/lib/redis", () => ({ getRedisClient: () => redisClientRef, })); +vi.mock("@/lib/redis/session-binding", () => ({ + ...bindingMocks, + getVersionedBindingCapabilityState: () => "unavailable", +})); + describe("SessionManager.terminateSession", () => { beforeEach(() => { vi.resetAllMocks(); @@ -39,6 +53,17 @@ describe("SessionManager.terminateSession", () => { eval: vi.fn(async () => 1), pipeline: vi.fn(() => pipelineRef), }; + bindingMocks.readOrReconcileSessionBinding.mockResolvedValue({ + status: "unavailable", + reason: "capability_unavailable", + capabilityState: "unavailable", + legacyFallbackAllowed: true, + }); + bindingMocks.mutateLegacySessionBindingSafely.mockResolvedValue({ + status: "ok", + changed: true, + providerId: null, + }); }); it("应同时从 global/key/user 的 active_sessions ZSET 中移除 sessionId（若可解析到 userId）", async () => { @@ -89,22 +114,185 @@ describe("SessionManager.terminateSession", () => { it("迟到 cleanup 仅删除仍绑定到预期 provider 的 session", async () => { const { SessionManager } = await import("@/lib/session-manager"); - await expect(SessionManager.clearSessionProvider("sess_compare", 42)).resolves.toBe(true); + await expect(SessionManager.clearSessionProvider("sess_compare", 42, 7)).resolves.toBe(true); - expect(redisClientRef.eval).toHaveBeenCalledWith( - expect.stringContaining('redis.call("GET", KEYS[1]) == ARGV[1]'), - 1, - "session:sess_compare:provider", - "42" + expect(bindingMocks.mutateLegacySessionBindingSafely).toHaveBeenCalledWith( + expect.objectContaining({ + sessionId: "sess_compare", + keyId: 7, + mutation: { type: "clear", expectedProviderId: 42 }, + }) ); - expect(redisClientRef.del).not.toHaveBeenCalled(); }); it("迟到 cleanup 不删除已切换到新 provider 的 session", async () => { - redisClientRef.eval.mockResolvedValueOnce(0); + bindingMocks.mutateLegacySessionBindingSafely.mockResolvedValueOnce({ + status: "conflict", + reason: "provider_mismatch", + legacyFallbackAllowed: false, + }); const { SessionManager } = await import("@/lib/session-manager"); - await expect(SessionManager.clearSessionProvider("sess_compare", 42)).resolves.toBe(false); + await expect(SessionManager.clearSessionProvider("sess_compare", 42, 7)).resolves.toBe(false); expect(redisClientRef.del).not.toHaveBeenCalled(); }); + + it("versioned termination leaves an owned tombstone instead of deleting mirrors", async () => { + const sessionId = "sess_versioned"; + redisClientRef.get.mockImplementation(async (key: string) => { + if (key === `session:${sessionId}:provider`) return "42"; + if (key === `session:${sessionId}:key`) return "7"; + return null; + }); + bindingMocks.readOrReconcileSessionBinding.mockResolvedValue({ + status: "ok", + source: "existing", + snapshot: { + sessionId, + keyId: 7, + providerId: 42, + generation: "generation-a", + }, + legacyFallbackAllowed: false, + }); + bindingMocks.terminateSessionBinding.mockResolvedValue({ + status: "ok", + source: "terminated", + snapshot: { + sessionId, + keyId: 7, + providerId: null, + generation: "generation-b", + }, + legacyFallbackAllowed: false, + }); + const { SessionManager } = await import("@/lib/session-manager"); + + await expect(SessionManager.terminateSession(sessionId)).resolves.toBe(true); + + expect(bindingMocks.terminateSessionBinding).toHaveBeenCalledWith( + expect.objectContaining({ + sessionId, + keyId: 7, + }) + ); + expect(pipelineRef.del).not.toHaveBeenCalledWith(`session:${sessionId}:provider`); + expect(pipelineRef.del).not.toHaveBeenCalledWith(`session:${sessionId}:key`); + }); + + it("preserves shared Session state after scoped versioned termination linearizes on the old Provider", async () => { + const sessionId = "sess_scoped_versioned_failover"; + redisClientRef.get.mockImplementation(async (key: string) => { + if (key === `session:${sessionId}:provider`) return "42"; + if (key === `session:${sessionId}:key`) return "7"; + return null; + }); + redisClientRef.hget.mockResolvedValue("123"); + bindingMocks.readOrReconcileSessionBinding.mockResolvedValue({ + status: "ok", + source: "existing", + snapshot: { + sessionId, + keyId: 7, + providerId: 42, + generation: "generation-before-provider-invalidation", + }, + legacyFallbackAllowed: false, + }); + bindingMocks.terminateSessionBinding.mockResolvedValue({ + status: "ok", + source: "terminated", + snapshot: { + sessionId, + keyId: 7, + providerId: null, + generation: "generation-after-provider-invalidation", + }, + legacyFallbackAllowed: false, + }); + const { getGlobalActiveSessionsKey, getKeyActiveSessionsKey, getUserActiveSessionsKey } = + await import("@/lib/redis/active-session-keys"); + const { SessionManager } = await import("@/lib/session-manager"); + + await expect(SessionManager.terminateSession(sessionId, [42])).resolves.toBe(true); + + expect(bindingMocks.terminateSessionBinding).toHaveBeenCalledWith( + expect.objectContaining({ + sessionId, + keyId: 7, + expectedProviderId: 42, + }) + ); + expect(pipelineRef.zrem).toHaveBeenCalledWith("provider:42:active_sessions", sessionId); + expect(pipelineRef.hdel).toHaveBeenCalledWith("provider:42:active_session_refs", sessionId); + expect(pipelineRef.del).not.toHaveBeenCalled(); + expect(pipelineRef.zrem).not.toHaveBeenCalledWith(getGlobalActiveSessionsKey(), sessionId); + expect(pipelineRef.zrem).not.toHaveBeenCalledWith(getKeyActiveSessionsKey(7), sessionId); + expect(pipelineRef.zrem).not.toHaveBeenCalledWith(getUserActiveSessionsKey(123), sessionId); + }); + + it("does not delete session metadata when versioned termination hits a mirror conflict", async () => { + const sessionId = "sess_mirror_conflict"; + redisClientRef.get.mockImplementation(async (key: string) => { + if (key === `session:${sessionId}:provider`) return "42"; + if (key === `session:${sessionId}:key`) return "7"; + return null; + }); + bindingMocks.readOrReconcileSessionBinding.mockResolvedValue({ + status: "ok", + source: "existing", + snapshot: { + sessionId, + keyId: 7, + providerId: 42, + generation: "generation-a", + }, + legacyFallbackAllowed: false, + }); + bindingMocks.terminateSessionBinding.mockResolvedValue({ + status: "conflict", + reason: "mirror_conflict", + legacyFallbackAllowed: false, + }); + const { SessionManager } = await import("@/lib/session-manager"); + + await expect(SessionManager.terminateSession(sessionId)).resolves.toBe(false); + expect(pipelineRef.exec).not.toHaveBeenCalled(); + expect(pipelineRef.del).not.toHaveBeenCalled(); + }); + + it("preserves shared Session state after scoped legacy termination linearizes on the old Provider", async () => { + const sessionId = "sess_scoped_legacy_failover"; + redisClientRef.get.mockImplementation(async (key: string) => { + if (key === `session:${sessionId}:provider`) return "42"; + if (key === `session:${sessionId}:key`) return "7"; + return null; + }); + redisClientRef.hget.mockResolvedValue("123"); + bindingMocks.mutateLegacySessionBindingSafely.mockResolvedValueOnce({ + status: "ok", + changed: true, + providerId: null, + terminatedProviderId: 42, + }); + const { getGlobalActiveSessionsKey, getKeyActiveSessionsKey, getUserActiveSessionsKey } = + await import("@/lib/redis/active-session-keys"); + const { SessionManager } = await import("@/lib/session-manager"); + + await expect(SessionManager.terminateSession(sessionId, [42])).resolves.toBe(true); + + expect(bindingMocks.mutateLegacySessionBindingSafely).toHaveBeenCalledWith( + expect.objectContaining({ + sessionId, + keyId: 7, + mutation: { type: "terminate", expectedProviderIds: [42] }, + }) + ); + expect(pipelineRef.zrem).toHaveBeenCalledWith("provider:42:active_sessions", sessionId); + expect(pipelineRef.hdel).toHaveBeenCalledWith("provider:42:active_session_refs", sessionId); + expect(pipelineRef.del).not.toHaveBeenCalled(); + expect(pipelineRef.zrem).not.toHaveBeenCalledWith(getGlobalActiveSessionsKey(), sessionId); + expect(pipelineRef.zrem).not.toHaveBeenCalledWith(getKeyActiveSessionsKey(7), sessionId); + expect(pipelineRef.zrem).not.toHaveBeenCalledWith(getUserActiveSessionsKey(123), sessionId); + }); }); diff --git a/tests/unit/lib/session-manager-versioned-binding.test.ts b/tests/unit/lib/session-manager-versioned-binding.test.ts new file mode 100644 index 000000000..ba09c068e --- /dev/null +++ b/tests/unit/lib/session-manager-versioned-binding.test.ts @@ -0,0 +1,279 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const bindingMocks = vi.hoisted(() => ({ + acquireSessionDiscoveryLease: vi.fn(), + buildSessionBindingKeys: vi.fn((sessionId: string, _keyId: number) => ({ + canonical: `session-binding:v1:{${"a".repeat(64)}}:binding`, + legacyProvider: `session:${sessionId}:provider`, + legacyOwner: `session:${sessionId}:key`, + })), + clearSessionBinding: vi.fn(), + compareAndSetSessionBinding: vi.fn(), + isSessionProviderCoolingDown: vi.fn(), + readOrReconcileSessionBinding: vi.fn(), + refreshSessionBinding: vi.fn(), + releaseSessionDiscoveryLease: vi.fn(), + renewSessionDiscoveryLease: vi.fn(), + touchSessionBinding: vi.fn(), +})); + +let redisClientRef: { + status: string; + del: ReturnType; + eval: ReturnType; + exists: ReturnType; + get: ReturnType; + pipeline: ReturnType; + set: ReturnType; + setex: ReturnType; +}; + +vi.mock("server-only", () => ({})); +vi.mock("@/lib/logger", () => ({ + logger: { + debug: vi.fn(), + error: vi.fn(), + info: vi.fn(), + trace: vi.fn(), + warn: vi.fn(), + }, +})); +vi.mock("@/lib/redis", () => ({ + getRedisClient: () => redisClientRef, +})); +vi.mock("@/lib/redis/session-binding", () => ({ + ...bindingMocks, + getVersionedBindingCapabilityState: () => "available", +})); + +import { SessionManager } from "@/lib/session-manager"; + +const SESSION_ID = "session-versioned"; +const KEY_ID = 7; +const PROVIDER_ID = 42; + +function snapshot(providerId: number | null = PROVIDER_ID) { + return { + status: "ok" as const, + source: "existing" as const, + snapshot: { + sessionId: SESSION_ID, + keyId: KEY_ID, + providerId, + generation: "generation-a", + }, + legacyFallbackAllowed: false as const, + }; +} + +beforeEach(() => { + vi.clearAllMocks(); + const pipeline = { + expire: vi.fn(), + exec: vi.fn(async () => []), + setex: vi.fn(), + }; + pipeline.expire.mockReturnValue(pipeline); + pipeline.setex.mockReturnValue(pipeline); + redisClientRef = { + status: "ready", + del: vi.fn(async () => 1), + eval: vi.fn(async () => 1), + exists: vi.fn(async () => 0), + get: vi.fn(async () => null), + pipeline: vi.fn(() => pipeline), + set: vi.fn(async () => "OK"), + setex: vi.fn(async () => "OK"), + }; + bindingMocks.readOrReconcileSessionBinding.mockResolvedValue(snapshot()); + bindingMocks.acquireSessionDiscoveryLease.mockResolvedValue({ + status: "acquired", + ownerToken: "owner-a", + legacyFallbackAllowed: false, + }); + bindingMocks.renewSessionDiscoveryLease.mockResolvedValue({ + status: "renewed", + legacyFallbackAllowed: false, + }); + bindingMocks.releaseSessionDiscoveryLease.mockResolvedValue({ + status: "released", + legacyFallbackAllowed: false, + }); + bindingMocks.touchSessionBinding.mockResolvedValue({ + ...snapshot(), + source: "touched", + }); +}); + +describe("SessionManager versioned binding adapter", () => { + it("delegates the complete tenant-scoped Discovery lease lifecycle", async () => { + await expect( + SessionManager.acquireSessionDiscoveryLease(SESSION_ID, KEY_ID, 61, "owner-a") + ).resolves.toMatchObject({ status: "acquired", ownerToken: "owner-a" }); + await expect( + SessionManager.renewSessionDiscoveryLease(SESSION_ID, KEY_ID, "owner-a", 61) + ).resolves.toMatchObject({ status: "renewed" }); + await expect( + SessionManager.releaseSessionDiscoveryLease(SESSION_ID, KEY_ID, "owner-a") + ).resolves.toMatchObject({ status: "released" }); + + expect(bindingMocks.acquireSessionDiscoveryLease).toHaveBeenCalledWith( + expect.objectContaining({ + sessionId: SESSION_ID, + keyId: KEY_ID, + ttlSeconds: 61, + ownerToken: "owner-a", + redis: redisClientRef, + }) + ); + expect(bindingMocks.renewSessionDiscoveryLease).toHaveBeenCalledWith( + expect.objectContaining({ + sessionId: SESSION_ID, + keyId: KEY_ID, + ttlSeconds: 61, + ownerToken: "owner-a", + redis: redisClientRef, + }) + ); + expect(bindingMocks.releaseSessionDiscoveryLease).toHaveBeenCalledWith( + expect.objectContaining({ + sessionId: SESSION_ID, + keyId: KEY_ID, + ownerToken: "owner-a", + redis: redisClientRef, + }) + ); + }); + + it("returns the tenant-scoped provider without reading the legacy mirror", async () => { + await expect(SessionManager.getSessionProvider(SESSION_ID, KEY_ID)).resolves.toBe(PROVIDER_ID); + + expect(bindingMocks.readOrReconcileSessionBinding).toHaveBeenCalledWith( + expect.objectContaining({ sessionId: SESSION_ID, keyId: KEY_ID }) + ); + expect(redisClientRef.get).not.toHaveBeenCalled(); + }); + + it("touches only the captured binding and exposes a TTL-derived heartbeat interval", async () => { + const binding = snapshot().snapshot; + const configuredTtl = Number.parseInt(process.env.SESSION_TTL || "300", 10); + + await expect(SessionManager.touchVersionedSessionBinding(binding)).resolves.toMatchObject({ + status: "ok", + source: "touched", + }); + + expect(bindingMocks.touchSessionBinding).toHaveBeenCalledWith({ + sessionId: SESSION_ID, + keyId: KEY_ID, + expectedGeneration: "generation-a", + expectedProviderId: PROVIDER_ID, + ttlSeconds: configuredTtl, + redis: redisClientRef, + }); + expect(SessionManager.getVersionedSessionBindingRefreshIntervalMs()).toBe( + Math.max(1, Math.floor((configuredTtl * 1000) / 3)) + ); + }); + + it("fails closed on a foreign legacy owner", async () => { + bindingMocks.readOrReconcileSessionBinding.mockResolvedValue({ + status: "conflict", + reason: "foreign_legacy_owner", + legacyFallbackAllowed: false, + }); + + await expect(SessionManager.getSessionProvider(SESSION_ID, KEY_ID)).resolves.toBeNull(); + + expect(redisClientRef.get).not.toHaveBeenCalled(); + }); + + it("does not reuse a legacy mirror when canonical state exists during capability fallback", async () => { + bindingMocks.readOrReconcileSessionBinding.mockResolvedValue({ + status: "unavailable", + reason: "capability_unavailable", + capabilityState: "unavailable", + legacyFallbackAllowed: true, + }); + redisClientRef.exists.mockResolvedValue(1); + redisClientRef.get.mockResolvedValue(String(PROVIDER_ID)); + + await expect(SessionManager.getSessionProvider(SESSION_ID, KEY_ID)).resolves.toBeNull(); + + expect(redisClientRef.exists).toHaveBeenCalledWith( + expect.stringMatching(/^session-binding:v1:\{[a-f0-9]+\}:binding$/) + ); + expect(redisClientRef.get).not.toHaveBeenCalled(); + }); + + it("still reuses a tenant-owned legacy-only mirror during capability fallback", async () => { + bindingMocks.readOrReconcileSessionBinding.mockResolvedValue({ + status: "unavailable", + reason: "capability_unavailable", + capabilityState: "unavailable", + legacyFallbackAllowed: true, + }); + redisClientRef.exists.mockResolvedValue(0); + redisClientRef.get.mockImplementation(async (key: string) => + key.endsWith(":key") ? String(KEY_ID) : String(PROVIDER_ID) + ); + + await expect(SessionManager.getSessionProvider(SESSION_ID, KEY_ID)).resolves.toBe(PROVIDER_ID); + + expect(redisClientRef.exists).toHaveBeenCalledWith( + expect.stringMatching(/^session-binding:v1:\{[a-f0-9]+\}:binding$/) + ); + expect(redisClientRef.get).toHaveBeenCalledWith(`session:${SESSION_ID}:key`); + expect(redisClientRef.get).toHaveBeenCalledWith(`session:${SESSION_ID}:provider`); + }); + + it("clears through generation CAS and does not delete the legacy provider directly", async () => { + bindingMocks.clearSessionBinding.mockResolvedValue({ + status: "ok", + source: "cleared", + snapshot: { + sessionId: SESSION_ID, + keyId: KEY_ID, + providerId: null, + generation: "generation-b", + }, + legacyFallbackAllowed: false, + }); + + await expect( + SessionManager.clearSessionProvider(SESSION_ID, PROVIDER_ID, KEY_ID) + ).resolves.toBe(true); + + expect(bindingMocks.clearSessionBinding).toHaveBeenCalledWith( + expect.objectContaining({ + sessionId: SESSION_ID, + keyId: KEY_ID, + expectedGeneration: "generation-a", + expectedProviderId: PROVIDER_ID, + }) + ); + expect(redisClientRef.del).not.toHaveBeenCalled(); + expect(redisClientRef.eval).not.toHaveBeenCalled(); + }); + + it("does not let a Codex cache key claim a foreign legacy session", async () => { + bindingMocks.readOrReconcileSessionBinding.mockResolvedValue({ + status: "conflict", + reason: "foreign_legacy_owner", + legacyFallbackAllowed: false, + }); + + await expect( + SessionManager.updateSessionWithCodexCacheKey( + "current-session", + "shared-cache-key", + PROVIDER_ID, + KEY_ID + ) + ).resolves.toEqual({ sessionId: "current-session", updated: false }); + + expect(redisClientRef.get).not.toHaveBeenCalled(); + expect(redisClientRef.pipeline).not.toHaveBeenCalled(); + expect(redisClientRef.setex).not.toHaveBeenCalled(); + }); +}); diff --git a/tests/unit/lib/session-tracker-cleanup.test.ts b/tests/unit/lib/session-tracker-cleanup.test.ts index e13f718ab..02a1cb046 100644 --- a/tests/unit/lib/session-tracker-cleanup.test.ts +++ b/tests/unit/lib/session-tracker-cleanup.test.ts @@ -74,7 +74,6 @@ describe("SessionTracker - TTL and cleanup", () => { pipelineCalls.length = 0; vi.useFakeTimers(); vi.setSystemTime(new Date(nowMs)); - redisClientRef = { status: "ready", exists: vi.fn(async () => 1), @@ -163,30 +162,17 @@ describe("SessionTracker - TTL and cleanup", () => { expect(providerExpireCall![2]).toBe(7200); // should use SESSION_TTL when > 3600 }); - it("should refresh session binding TTLs using env SESSION_TTL (not hardcoded 300)", async () => { + it("should refresh session activity using env SESSION_TTL (not hardcoded 300)", async () => { process.env.SESSION_TTL = "600"; // 10 minutes const { SessionTracker } = await import("@/lib/session-tracker"); await SessionTracker.refreshSession("sess-123", 1, 42); - // Check expire calls for session bindings use 600 (env value), not 300 - const providerBindingExpire = pipelineCalls.find( - (call) => call[0] === "expire" && String(call[1]) === "session:sess-123:provider" - ); - const keyBindingExpire = pipelineCalls.find( - (call) => call[0] === "expire" && String(call[1]) === "session:sess-123:key" - ); const lastSeenSetex = pipelineCalls.find( (call) => call[0] === "setex" && String(call[1]) === "session:sess-123:last_seen" ); - expect(providerBindingExpire).toBeDefined(); - expect(providerBindingExpire![2]).toBe(600); - - expect(keyBindingExpire).toBeDefined(); - expect(keyBindingExpire![2]).toBe(600); - expect(lastSeenSetex).toBeDefined(); expect(lastSeenSetex![2]).toBe(600); }); diff --git a/tests/unit/proxy/provider-selector-cross-type-model.test.ts b/tests/unit/proxy/provider-selector-cross-type-model.test.ts index f171f6ffe..94eaec709 100644 --- a/tests/unit/proxy/provider-selector-cross-type-model.test.ts +++ b/tests/unit/proxy/provider-selector-cross-type-model.test.ts @@ -303,7 +303,8 @@ describe("findReusable - cross-type model routing (#832)", () => { expect(result).toBeNull(); expect(sessionManagerMocks.SessionManager.clearSessionProvider).toHaveBeenCalledWith( "cross-type-3", - 12 + 12, + null ); }); @@ -336,7 +337,8 @@ describe("findReusable - cross-type model routing (#832)", () => { expect(result).toBeNull(); expect(sessionManagerMocks.SessionManager.clearSessionProvider).toHaveBeenCalledWith( "cross-type-6", - 15 + 15, + null ); }); }); diff --git a/tests/unit/proxy/provider-selector-model-mismatch-binding.test.ts b/tests/unit/proxy/provider-selector-model-mismatch-binding.test.ts index d21486433..5efbe753a 100644 --- a/tests/unit/proxy/provider-selector-model-mismatch-binding.test.ts +++ b/tests/unit/proxy/provider-selector-model-mismatch-binding.test.ts @@ -116,7 +116,8 @@ describe("findReusable - model mismatch clears stale binding", () => { expect(result).toBeNull(); expect(sessionManagerMocks.SessionManager.clearSessionProvider).toHaveBeenCalledWith( "sess_disable_reuse", - 78 + 78, + null ); }); @@ -141,7 +142,8 @@ describe("findReusable - model mismatch clears stale binding", () => { // Key assertion: clearSessionProvider should have been called expect(sessionManagerMocks.SessionManager.clearSessionProvider).toHaveBeenCalledWith( "4c25cf92", - 78 + 78, + null ); }); @@ -165,7 +167,8 @@ describe("findReusable - model mismatch clears stale binding", () => { expect(result).toBeNull(); expect(sessionManagerMocks.SessionManager.clearSessionProvider).toHaveBeenCalledWith( "sess_response_format_mismatch", - 94 + 94, + null ); }); @@ -240,7 +243,8 @@ describe("findReusable - model mismatch clears stale binding", () => { expect(result).toBeNull(); expect(sessionManagerMocks.SessionManager.clearSessionProvider).toHaveBeenCalledWith( "sess_variant", - 78 + 78, + null ); }); }); diff --git a/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts b/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts index bfa2dc845..6322286c2 100644 --- a/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts +++ b/tests/unit/proxy/proxy-forwarder-hedge-first-byte.test.ts @@ -13,6 +13,7 @@ const mocks = vi.hoisted(() => ({ updateSessionBindingSmart: vi.fn(async () => ({ updated: true, reason: "test" })), updateSessionProvider: vi.fn(async () => {}), clearSessionProvider: vi.fn(async () => {}), + clearSessionProviders: vi.fn(async () => false), isHttp2Enabled: vi.fn(async () => false), getPreferredProviderEndpoints: vi.fn(async () => []), getEndpointFilterStats: vi.fn(async () => null), @@ -92,6 +93,7 @@ vi.mock("@/lib/session-manager", () => ({ updateSessionBindingSmart: mocks.updateSessionBindingSmart, updateSessionProvider: mocks.updateSessionProvider, clearSessionProvider: mocks.clearSessionProvider, + clearSessionProviders: mocks.clearSessionProviders, storeSessionSpecialSettings: mocks.storeSessionSpecialSettings, storeSessionRequestPhaseSnapshot: mocks.storeSessionRequestPhaseSnapshot, storeSessionResponsePhaseSnapshot: mocks.storeSessionResponsePhaseSnapshot, @@ -121,6 +123,7 @@ import { import { ProxyForwarder } from "@/app/v1/_lib/proxy/forwarder"; import { ModelRedirector } from "@/app/v1/_lib/proxy/model-redirector"; import { ProxySession } from "@/app/v1/_lib/proxy/session"; +import { peekDeferredStreamingFinalization } from "@/app/v1/_lib/proxy/stream-finalization"; import { logger } from "@/lib/logger"; import type { Provider } from "@/types/provider"; @@ -920,7 +923,22 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { const provider1 = createProvider({ id: 1, name: "p1", firstByteTimeoutStreamingMs: 100 }); const provider2 = createProvider({ id: 2, name: "p2", firstByteTimeoutStreamingMs: 100 }); const session = createSession(); + session.authState = { + ...session.authState!, + key: { id: 456 }, + } as never; setProviderWithSessionRef(session, provider1); + const winnerBindingSnapshot = { + sessionId: "sess-hedge", + keyId: 456, + providerId: 2, + generation: "hedge-winner-generation", + }; + mocks.updateSessionBindingSmart.mockResolvedValueOnce({ + updated: true, + reason: "race_winner_forced", + bindingSnapshot: winnerBindingSnapshot, + } as never); mocks.pickRandomProviderWithExclusion.mockResolvedValueOnce(provider2); @@ -963,7 +981,12 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { await vi.advanceTimersByTimeAsync(50); const response = await responsePromise; + const deferred = peekDeferredStreamingFinalization(session); expect(await response.text()).toContain('"provider":"p2"'); + await expect(deferred?.hedgeBindingAuthorityPromise).resolves.toEqual({ + snapshot: winnerBindingSnapshot, + legacyClearAllowed: false, + }); expect(controller1.signal.aborted).toBe(true); expect(controller2.signal.aborted).toBe(false); expect(mocks.recordFailure).not.toHaveBeenCalled(); @@ -977,7 +1000,7 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { 0, false, true, - null, + 456, true ); expect(mocks.releaseProviderSession).toHaveBeenCalledWith(1, "sess-hedge"); @@ -1454,7 +1477,7 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { await rejection; expect(controller1.signal.aborted).toBe(true); expect(controller2.signal.aborted).toBe(true); - expect(mocks.clearSessionProvider).toHaveBeenCalledWith("sess-hedge", 1); + expect(mocks.clearSessionProviders).toHaveBeenCalledWith("sess-hedge", new Set([1, 2]), null); expect(mocks.recordFailure).not.toHaveBeenCalled(); expect(mocks.recordSuccess).not.toHaveBeenCalled(); @@ -1726,7 +1749,11 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { expect(error.message).toBe("所有供应商暂时不可用，请稍后重试"); expect(error.message).not.toContain("invalid key"); expect(error.message).not.toContain("model not found"); - expect(mocks.clearSessionProvider).toHaveBeenCalledWith("sess-hedge", 1); + expect(mocks.clearSessionProviders).toHaveBeenCalledWith( + "sess-hedge", + new Set([1, 2]), + null + ); } finally { vi.useRealTimers(); } @@ -1774,7 +1801,7 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { expect(error.message).toBe("prompt too long"); expect(doForward).toHaveBeenCalledTimes(1); expect(mocks.pickRandomProviderWithExclusion).not.toHaveBeenCalled(); - expect(mocks.clearSessionProvider).toHaveBeenCalledWith("sess-hedge", 1); + expect(mocks.clearSessionProviders).toHaveBeenCalledWith("sess-hedge", new Set([1]), null); expect(session.getProviderChain()).toEqual( expect.arrayContaining([ expect.objectContaining({ @@ -1831,7 +1858,7 @@ describe("ProxyForwarder - first-byte hedge scheduling", () => { expect(mocks.pickRandomProviderWithExclusion).not.toHaveBeenCalled(); expect(mocks.recordEndpointFailure).not.toHaveBeenCalled(); expect(mocks.recordFailure).not.toHaveBeenCalled(); - expect(mocks.clearSessionProvider).toHaveBeenCalledWith("sess-hedge", 1); + expect(mocks.clearSessionProviders).toHaveBeenCalledWith("sess-hedge", new Set([1]), null); expect(session.getProviderChain()).toEqual([ expect.objectContaining({ id: provider.id, diff --git a/tests/unit/proxy/response-handler-endpoint-circuit-isolation.test.ts b/tests/unit/proxy/response-handler-endpoint-circuit-isolation.test.ts index c88b61ff6..9960a59c9 100644 --- a/tests/unit/proxy/response-handler-endpoint-circuit-isolation.test.ts +++ b/tests/unit/proxy/response-handler-endpoint-circuit-isolation.test.ts @@ -74,10 +74,13 @@ vi.mock("@/repository/message", () => ({ vi.mock("@/lib/session-manager", () => ({ SessionManager: { + clearVersionedSessionProvider: vi.fn(), updateSessionUsage: vi.fn(), storeSessionResponse: vi.fn(), clearSessionProvider: vi.fn(), extractCodexPromptCacheKey: vi.fn(), + getVersionedSessionBindingRefreshIntervalMs: vi.fn(), + touchVersionedSessionBinding: vi.fn(), updateSessionBindingSmart: vi.fn(), updateSessionProvider: vi.fn(), updateSessionWithCodexCacheKey: vi.fn(), @@ -272,7 +275,11 @@ function createSession(opts?: { sessionId?: string | null }): ProxySession { return session; } -function setDeferredMeta(session: ProxySession, endpointId: number | null = 42) { +function setDeferredMeta( + session: ProxySession, + endpointId: number | null = 42, + extra: Partial[1]> = {} +) { setDeferredStreamingFinalization(session, { providerId: 1, providerName: "test-provider", @@ -284,6 +291,7 @@ function setDeferredMeta(session: ProxySession, endpointId: number | null = 42) endpointId, endpointUrl: "https://api.test.com", upstreamStatusCode: 200, + ...extra, }); } @@ -366,6 +374,31 @@ function createSuccessStreamResponse(): Response { }); } +function createControllableSuccessStreamResponse(): { + response: Response; + complete: () => void; +} { + const encoder = new TextEncoder(); + let sourceController: ReadableStreamDefaultController | null = null; + const stream = new ReadableStream({ + start(controller) { + sourceController = controller; + controller.enqueue( + encoder.encode( + `event: message_delta\ndata: ${JSON.stringify({ usage: { input_tokens: 100, output_tokens: 50 } })}\n\n` + ) + ); + }, + }); + return { + response: new Response(stream, { + status: 200, + headers: { "content-type": "text/event-stream" }, + }), + complete: () => sourceController?.close(), + }; +} + async function drainAsyncTasks(): Promise { while (asyncTasks.length > 0) { const tasks = asyncTasks.splice(0, asyncTasks.length); @@ -398,7 +431,24 @@ function setupCommonMocks() { vi.mocked(SessionManager.updateSessionUsage).mockResolvedValue(undefined); vi.mocked(SessionManager.storeSessionResponse).mockResolvedValue(undefined); vi.mocked(SessionManager.clearSessionProvider).mockResolvedValue(undefined); - vi.mocked(SessionManager.updateSessionUsage).mockResolvedValue(undefined); + vi.mocked(SessionManager.clearVersionedSessionProvider).mockResolvedValue({ + status: "ok", + source: "cleared", + snapshot: { + sessionId: "fake-session", + keyId: 456, + providerId: null, + generation: "cleared-generation", + }, + legacyFallbackAllowed: false, + }); + vi.mocked(SessionManager.getVersionedSessionBindingRefreshIntervalMs).mockReturnValue(100_000); + vi.mocked(SessionManager.touchVersionedSessionBinding).mockImplementation(async (snapshot) => ({ + status: "ok", + source: "touched", + snapshot, + legacyFallbackAllowed: false, + })); vi.mocked(SessionManager.updateSessionBindingSmart).mockResolvedValue({ updated: true, reason: "test", @@ -446,7 +496,7 @@ describe("Endpoint circuit breaker isolation", () => { expect.objectContaining({ message: expect.stringContaining("FAKE_200") }) ); expect(mockRecordEndpointFailure).not.toHaveBeenCalled(); - expect(SessionManager.clearSessionProvider).toHaveBeenCalledWith("fake-session", 1); + expect(SessionManager.clearSessionProvider).toHaveBeenCalledWith("fake-session", 1, 456); const chain = session.getProviderChain(); expect( @@ -475,7 +525,7 @@ describe("Endpoint circuit breaker isolation", () => { ); expect(mockRecordEndpointSuccess).not.toHaveBeenCalled(); expect(mockRecordEndpointFailure).not.toHaveBeenCalled(); - expect(SessionManager.clearSessionProvider).toHaveBeenCalledWith("fake-session", 1); + expect(SessionManager.clearSessionProvider).toHaveBeenCalledWith("fake-session", 1, 456); expect(updateMessageRequestDetailsDurably).toHaveBeenCalledWith( 1, expect.objectContaining({ @@ -504,7 +554,7 @@ describe("Endpoint circuit breaker isolation", () => { expect.objectContaining({ message: expect.stringContaining("FAKE_200") }) ); expect(mockRecordEndpointFailure).not.toHaveBeenCalled(); - expect(SessionManager.clearSessionProvider).toHaveBeenCalledWith("fake-session", 1); + expect(SessionManager.clearSessionProvider).toHaveBeenCalledWith("fake-session", 1, 456); expect(SessionManager.updateSessionUsage).not.toHaveBeenCalled(); expect(SessionTracker.refreshSession).not.toHaveBeenCalled(); }); @@ -520,7 +570,7 @@ describe("Endpoint circuit breaker isolation", () => { expect(mockRecordFailure).not.toHaveBeenCalled(); expect(mockRecordEndpointFailure).not.toHaveBeenCalled(); - expect(SessionManager.clearSessionProvider).toHaveBeenCalledWith("fake-session", 1); + expect(SessionManager.clearSessionProvider).toHaveBeenCalledWith("fake-session", 1, 456); const chain = session.getProviderChain(); expect( @@ -598,4 +648,187 @@ describe("Endpoint circuit breaker isolation", () => { expect(mockRecordEndpointSuccess).not.toHaveBeenCalled(); expect(mockRecordEndpointFailure).not.toHaveBeenCalled(); }); + + it("keeps the captured legacy Hedge winner binding alive throughout a long stream", async () => { + vi.useFakeTimers(); + try { + const session = createSession(); + const snapshot = { + sessionId: "fake-session", + keyId: 456, + providerId: 1, + generation: "hedge-long-stream-generation", + } as const; + setDeferredMeta(session, 42, { + isHedgeWinner: true, + hedgeBindingAuthorityPromise: Promise.resolve({ + snapshot, + legacyClearAllowed: false, + }), + }); + vi.mocked(SessionManager.getVersionedSessionBindingRefreshIntervalMs).mockReturnValue(1_000); + const controlled = createControllableSuccessStreamResponse(); + + const clientResponse = await ProxyResponseHandler.dispatch(session, controlled.response); + const bodyPromise = clientResponse.text(); + await vi.advanceTimersByTimeAsync(3_000); + + // Immediate ownership validation plus one heartbeat per interval. + expect(SessionManager.touchVersionedSessionBinding).toHaveBeenCalledTimes(4); + expect(SessionManager.touchVersionedSessionBinding).toHaveBeenLastCalledWith(snapshot); + + controlled.complete(); + await bodyPromise; + await drainAsyncTasks(); + + // A final touch gives the next turn a complete TTL from stream completion. + expect(SessionManager.touchVersionedSessionBinding).toHaveBeenCalledTimes(5); + expect(SessionManager.touchVersionedSessionBinding).toHaveBeenLastCalledWith(snapshot); + expect(SessionManager.updateSessionBindingSmart).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(3_000); + expect(SessionManager.touchVersionedSessionBinding).toHaveBeenCalledTimes(5); + } finally { + vi.useRealTimers(); + } + }); + + it("clears a failed Hedge stream only through its captured generation", async () => { + vi.useFakeTimers(); + try { + const session = createSession(); + const snapshot = { + sessionId: "fake-session", + keyId: 456, + providerId: 1, + generation: "failed-hedge-generation", + } as const; + setDeferredMeta(session, 42, { + isHedgeWinner: true, + hedgeBindingAuthorityPromise: Promise.resolve({ + snapshot, + legacyClearAllowed: false, + }), + }); + vi.mocked(SessionManager.getVersionedSessionBindingRefreshIntervalMs).mockReturnValue(1_000); + vi.mocked(SessionManager.clearVersionedSessionProvider).mockResolvedValueOnce({ + status: "conflict", + reason: "generation_mismatch", + legacyFallbackAllowed: false, + }); + + const clientResponse = await ProxyResponseHandler.dispatch( + session, + createFake200StreamResponse() + ); + await clientResponse.text(); + await drainAsyncTasks(); + + expect(SessionManager.clearVersionedSessionProvider).toHaveBeenCalledWith(snapshot, 1); + expect(SessionManager.clearSessionProvider).not.toHaveBeenCalled(); + const touchesAfterFailure = vi.mocked(SessionManager.touchVersionedSessionBinding).mock.calls + .length; + await vi.advanceTimersByTimeAsync(5_000); + expect(SessionManager.touchVersionedSessionBinding).toHaveBeenCalledTimes( + touchesAfterFailure + ); + } finally { + vi.useRealTimers(); + } + }); + + it("does not let a failed stale Hedge stream clear a newer same-Provider generation", async () => { + const session = createSession(); + setDeferredMeta(session, 42, { + isHedgeWinner: true, + // The first-byte write read G0, but a concurrent request established + // Provider 1 at G1 before its CAS. No authority over G1 was acquired. + hedgeBindingAuthorityPromise: Promise.resolve({ + snapshot: null, + legacyClearAllowed: false, + }), + }); + + const clientResponse = await ProxyResponseHandler.dispatch( + session, + createFake200StreamResponse() + ); + await clientResponse.text(); + await drainAsyncTasks(); + + expect(SessionManager.clearVersionedSessionProvider).not.toHaveBeenCalled(); + expect(SessionManager.clearSessionProvider).not.toHaveBeenCalled(); + expect(SessionManager.touchVersionedSessionBinding).not.toHaveBeenCalled(); + }); + + it("uses generic cleanup only after the Hedge winner confirms a legacy binding write", async () => { + const session = createSession(); + setDeferredMeta(session, 42, { + isHedgeWinner: true, + hedgeBindingAuthorityPromise: Promise.resolve({ + snapshot: null, + legacyClearAllowed: true, + }), + }); + + const clientResponse = await ProxyResponseHandler.dispatch( + session, + createFake200StreamResponse() + ); + await clientResponse.text(); + await drainAsyncTasks(); + + expect(SessionManager.clearVersionedSessionProvider).not.toHaveBeenCalled(); + expect(SessionManager.clearSessionProvider).toHaveBeenCalledWith("fake-session", 1, 456); + expect(SessionManager.touchVersionedSessionBinding).not.toHaveBeenCalled(); + }); + + it("stops Hedge heartbeats after a generation conflict and never revives the binding", async () => { + vi.useFakeTimers(); + try { + const session = createSession(); + const snapshot = { + sessionId: "fake-session", + keyId: 456, + providerId: 1, + generation: "generation-before-termination", + } as const; + setDeferredMeta(session, 42, { + isHedgeWinner: true, + hedgeBindingAuthorityPromise: Promise.resolve({ + snapshot, + legacyClearAllowed: false, + }), + }); + vi.mocked(SessionManager.getVersionedSessionBindingRefreshIntervalMs).mockReturnValue(1_000); + vi.mocked(SessionManager.touchVersionedSessionBinding) + .mockResolvedValueOnce({ + status: "ok", + source: "touched", + snapshot, + legacyFallbackAllowed: false, + }) + .mockResolvedValueOnce({ + status: "conflict", + reason: "generation_mismatch", + legacyFallbackAllowed: false, + }); + const controlled = createControllableSuccessStreamResponse(); + + const clientResponse = await ProxyResponseHandler.dispatch(session, controlled.response); + const bodyPromise = clientResponse.text(); + await vi.advanceTimersByTimeAsync(5_000); + expect(SessionManager.touchVersionedSessionBinding).toHaveBeenCalledTimes(2); + + controlled.complete(); + await bodyPromise; + await drainAsyncTasks(); + + // Completion must not retry after an administrator or concurrent request + // advances the generation. + expect(SessionManager.touchVersionedSessionBinding).toHaveBeenCalledTimes(2); + expect(SessionManager.updateSessionBindingSmart).not.toHaveBeenCalled(); + } finally { + vi.useRealTimers(); + } + }); });