diff --git a/.changeset/local-model-vram-abort-1753.md b/.changeset/local-model-vram-abort-1753.md new file mode 100644 index 00000000..6907dae6 --- /dev/null +++ b/.changeset/local-model-vram-abort-1753.md @@ -0,0 +1,5 @@ +--- +"secureflow": patch +--- + +fix(ai): wire request abort signals into local model generation and enforce semaphore-based VRAM concurrency queue (#1753) diff --git a/src/ai/flows/security-explanation-core.ts b/src/ai/flows/security-explanation-core.ts index 5e54ac14..67c24b8e 100644 --- a/src/ai/flows/security-explanation-core.ts +++ b/src/ai/flows/security-explanation-core.ts @@ -1,7 +1,7 @@ import { InsufficientVRAMError, isInsufficientMemoryError } from "@/ai/local-model"; import { getModelId } from "@/lib/explanation-cache"; import { isRateLimitError, isTimeoutError } from "./security-helpers"; -import { getModelCircuitBreaker, CircuitBreakerError, CircuitState } from "@/ai/resilience"; +import { getModelCircuitBreaker, CircuitBreakerError, CircuitState, isVramConcurrencyError } from "@/ai/resilience"; import { AISecurityExplanationOutputSchema, type AISecurityExplanationInput, @@ -18,7 +18,11 @@ export interface TokenUsage { } export type FallbackReason = - "rate_limit" | "timeout" | "insufficient_memory" | "circuit_breaker_open"; + | "rate_limit" + | "timeout" + | "insufficient_memory" + | "circuit_breaker_open" + | "vram_concurrency_exceeded"; export function prepareExplanation(input: AISecurityExplanationInput) { return { @@ -82,6 +86,25 @@ export async function runModelChain( continue; } + if (isVramConcurrencyError(error)) { + if (next === undefined) { + throw error; + } + + console.warn( + `[AI_FALLBACK] Model ${String(model)} exceeded VRAM concurrency limit. Switching to ${String(next)}`, + ); + + onFallback?.({ + from: model, + to: next, + reason: "vram_concurrency_exceeded", + error, + }); + + continue; + } + if (isInsufficientMemoryError(error)) { if (next === undefined) { throw new InsufficientVRAMError(models.map((m) => getModelId(m as any))); diff --git a/src/ai/local-model.test.ts b/src/ai/local-model.test.ts index 90d0d378..615d02bc 100644 --- a/src/ai/local-model.test.ts +++ b/src/ai/local-model.test.ts @@ -13,6 +13,8 @@ import { DEFAULT_VLLM_MODEL, DEFAULT_LOCAL_AI_URL, DEFAULT_LOCAL_AI_MODEL, + streamLocalModelGeneration, + generateLocalModel, } from "./local-model"; // --------------------------------------------------------------------------- @@ -434,3 +436,121 @@ describe("default constants", () => { expect(DEFAULT_VLLM_MODEL).toBe("meta-llama/Meta-Llama-3-8B-Instruct"); }); }); + + +// --------------------------------------------------------------------------- +// streamLocalModelGeneration & Abort Signal Wire-up (#1753) +// --------------------------------------------------------------------------- + +describe("streamLocalModelGeneration & generateLocalModel (#1753)", () => { + it("returns immediately without fetching if signal is already aborted", async () => { + const controller = new AbortController(); + controller.abort(); + + const fetchSpy = vi.spyOn(globalThis, "fetch"); + + const tokens: string[] = []; + for await (const chunk of streamLocalModelGeneration({ + prompt: "test", + signal: controller.signal, + })) { + tokens.push(chunk); + } + + expect(tokens).toEqual([]); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + it("streams tokens from OpenAI-compatible SSE chunks", async () => { + const sseData = + 'data: {"choices":[{"delta":{"content":"Hello"}}]}\n\n' + + 'data: {"choices":[{"delta":{"content":" world"}}]}\n\n' + + 'data: [DONE]\n\n'; + + const encoder = new TextEncoder(); + const stream = new ReadableStream({ + start(ctrl) { + ctrl.enqueue(encoder.encode(sseData)); + ctrl.close(); + }, + }); + + vi.spyOn(globalThis, "fetch").mockResolvedValueOnce({ + ok: true, + status: 200, + body: stream, + } as any); + + const tokens: string[] = []; + for await (const token of streamLocalModelGeneration({ + prompt: "hi", + config: { baseUrl: "http://localhost:11434/v1", model: "llama3" }, + })) { + tokens.push(token); + } + + expect(tokens).toEqual(["Hello", " world"]); + }); + + it("halts token generation and cancels reader immediately on client abort signal", async () => { + const controller = new AbortController(); + let readerCancelled = false; + + const encoder = new TextEncoder(); + const stream = new ReadableStream({ + start(ctrl) { + ctrl.enqueue(encoder.encode('data: {"choices":[{"delta":{"content":"Token 1"}}]}\n\n')); + }, + cancel() { + readerCancelled = true; + }, + }); + + vi.spyOn(globalThis, "fetch").mockResolvedValueOnce({ + ok: true, + status: 200, + body: stream, + } as any); + + const tokens: string[] = []; + for await (const token of streamLocalModelGeneration({ + prompt: "abort test", + signal: controller.signal, + config: { baseUrl: "http://localhost:11434/v1", model: "llama3" }, + })) { + tokens.push(token); + controller.abort(); + } + + expect(tokens).toEqual(["Token 1"]); + expect(readerCancelled).toBe(true); + }); + + it("generateLocalModel accumulates all streamed tokens into complete text", async () => { + const sseData = + 'data: {"choices":[{"delta":{"content":"Secure"}}]}\n\n' + + 'data: {"choices":[{"delta":{"content":"Flow"}}]}\n\n' + + 'data: [DONE]\n\n'; + + const encoder = new TextEncoder(); + const stream = new ReadableStream({ + start(ctrl) { + ctrl.enqueue(encoder.encode(sseData)); + ctrl.close(); + }, + }); + + vi.spyOn(globalThis, "fetch").mockResolvedValueOnce({ + ok: true, + status: 200, + body: stream, + } as any); + + const result = await generateLocalModel({ + prompt: "test accumulation", + config: { baseUrl: "http://localhost:11434/v1", model: "llama3" }, + }); + + expect(result.text).toBe("SecureFlow"); + }); +}); \ No newline at end of file diff --git a/src/ai/local-model.ts b/src/ai/local-model.ts index fdc85cd6..45d65d13 100644 --- a/src/ai/local-model.ts +++ b/src/ai/local-model.ts @@ -364,6 +364,158 @@ export class InsufficientVRAMError extends Error { } } + +// --------------------------------------------------------------------------- +// Local Model Generation & Streaming Loops with Abort Support (#1753) +// --------------------------------------------------------------------------- + +export interface LocalGenerationOptions { + prompt: string; + system?: string; + model?: string; + config?: LocalModelConfig; + signal?: AbortSignal; + temperature?: number; + maxTokens?: number; +} + +/** + * Streams tokens from a local OpenAI-compatible inference server (Ollama, vLLM). + * Wires the caller's AbortSignal directly into the fetch call and reader loop + * so client disconnects halt GPU token generation immediately without pinning VRAM (#1753). + */ +export async function* streamLocalModelGeneration( + options: LocalGenerationOptions, +): AsyncGenerator { + const signal = options.signal; + if (signal?.aborted) return; + + const config = + options.config || + resolveLocalModelConfig() || { + baseUrl: DEFAULT_LOCAL_AI_URL, + model: options.model || DEFAULT_LOCAL_AI_MODEL, + provider: "ollama", + apiKey: "local", + }; + + const endpoint = `${config.baseUrl}/chat/completions`; + const body = { + model: options.model || config.model, + messages: [ + ...(options.system ? [{ role: "system", content: options.system }] : []), + { role: "user", content: options.prompt }, + ], + stream: true, + temperature: options.temperature ?? 0.1, + max_tokens: options.maxTokens ?? 1024, + }; + + let response: Response; + try { + response = await fetch(endpoint, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${config.apiKey || "local"}`, + }, + body: JSON.stringify(body), + signal, + }); + } catch (err: any) { + if (signal?.aborted || err?.name === "AbortError") { + return; + } + throw err; + } + + if (!response.ok) { + const errorText = await response.text().catch(() => ""); + throw new Error(`Local model request failed with status ${response.status}: ${errorText}`); + } + + const reader = response.body?.getReader(); + if (!reader) return; + + const decoder = new TextDecoder(); + let buffer = ""; + + const onAbort = () => { + reader.cancel().catch(() => {}); + }; + if (signal) { + signal.addEventListener("abort", onAbort, { once: true }); + } + + try { + while (true) { + if (signal?.aborted) { + await reader.cancel().catch(() => {}); + return; + } + + let readResult: ReadableStreamReadResult; + try { + readResult = await reader.read(); + } catch (readErr: any) { + if (signal?.aborted || readErr?.name === "AbortError") { + return; + } + throw readErr; + } + + const { done, value } = readResult; + if (done) break; + + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split("\n"); + buffer = lines.pop() ?? ""; + + for (const line of lines) { + if (signal?.aborted) { + await reader.cancel().catch(() => {}); + return; + } + + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith(":")) continue; + if (trimmed === "data: [DONE]") return; + if (trimmed.startsWith("data: ")) { + try { + const data = JSON.parse(trimmed.slice(6)); + const content = data.choices?.[0]?.delta?.content ?? ""; + if (content) { + yield content; + } + } catch { + // Ignore unparseable partial chunk + } + } + } + } + } finally { + if (signal && onAbort) { + signal.removeEventListener("abort", onAbort); + } + try { + await reader.cancel(); + } catch {} + } +} + +/** + * Runs a local model completion with direct abort signal support (#1753). + */ +export async function generateLocalModel( + options: LocalGenerationOptions, +): Promise<{ text: string }> { + let fullText = ""; + for await (const chunk of streamLocalModelGeneration(options)) { + fullText += chunk; + } + return { text: fullText }; +} + // ── ArmorIQ Pre-Execution Intent Signing (#1275) ────────────────────────── export { declareAndSignIntent as emitExecutionPlan, diff --git a/src/ai/resilience.test.ts b/src/ai/resilience.test.ts index 9c6c559f..ed7d3027 100644 --- a/src/ai/resilience.test.ts +++ b/src/ai/resilience.test.ts @@ -22,6 +22,12 @@ import { tripModelCircuitBreaker, CircuitState, CircuitBreakerError, + Semaphore, + VRAMConcurrencyError, + isVramConcurrencyError, + withModelConcurrency, + getLocalModelSemaphore, + resetLocalModelSemaphore, } from "./resilience"; const rateLimit = () => Object.assign(new Error("Too many requests"), { status: 429 }); @@ -399,4 +405,145 @@ describe("executeWithFallbackAndRetry", () => { expect(replicaBreaker.getState()).toBe(CircuitState.CLOSED); }); }); + describe("Semaphore & VRAM Concurrency Control (#1753)", () => { + beforeEach(() => { + vi.useRealTimers(); + }); + + afterEach(() => { + resetLocalModelSemaphore(); + }); + + it("acquires and releases permits within capacity", async () => { + const sem = new Semaphore(2); + expect(sem.availablePermits).toBe(2); + + const release1 = await sem.acquire(); + expect(sem.availablePermits).toBe(1); + + const release2 = await sem.acquire(); + expect(sem.availablePermits).toBe(0); + + release1(); + expect(sem.availablePermits).toBe(1); + + release2(); + expect(sem.availablePermits).toBe(2); + }); + + it("queues requests in FIFO order when permits are exhausted", async () => { + const sem = new Semaphore(1); + const order: number[] = []; + + const release1 = await sem.acquire(); + order.push(1); + + const p2 = sem.acquire().then((rel) => { + order.push(2); + rel(); + }); + + const p3 = sem.acquire().then((rel) => { + order.push(3); + rel(); + }); + + expect(sem.queueLength).toBe(2); + release1(); + + await Promise.all([p2, p3]); + expect(order).toEqual([1, 2, 3]); + expect(sem.availablePermits).toBe(1); + }); + + it("times out and throws VRAMConcurrencyError when queue wait exceeds timeoutMs", async () => { + const sem = new Semaphore(1); + const release = await sem.acquire(); + + const acquirePromise = sem.acquire({ timeoutMs: 50 }); + await expect(acquirePromise).rejects.toThrow(VRAMConcurrencyError); + expect(sem.queueLength).toBe(0); + + release(); + expect(sem.availablePermits).toBe(1); + }); + + it("rejects with AbortError when signal is aborted while in queue", async () => { + const sem = new Semaphore(1); + const release = await sem.acquire(); + + const controller = new AbortController(); + const acquirePromise = sem.acquire({ signal: controller.signal }); + + controller.abort(); + await expect(acquirePromise).rejects.toThrow("Operation aborted"); + expect(sem.queueLength).toBe(0); + + release(); + }); + + it("withModelConcurrency runs action within permit boundary and releases on error", async () => { + const sem = new Semaphore(1); + await expect( + withModelConcurrency( + async () => { + expect(sem.availablePermits).toBe(0); + throw new Error("fail inside permit"); + }, + { semaphore: sem }, + ), + ).rejects.toThrow("fail inside permit"); + + expect(sem.availablePermits).toBe(1); + }); + + it("fails over cleanly to fallback model on VRAMConcurrencyError", async () => { + const sem = new Semaphore(1); + const blockRelease = await sem.acquire(); + + const op = vi.fn(async (model: string) => { + if (model === "openai/llama3") { + return "local-result"; + } + return "cloud-fallback-result"; + }); + + const onModelSwitch = vi.fn(); + const { result, stats } = await executeWithFallbackAndRetry(op, { + primaryModel: "openai/llama3", + fallbackModels: ["groq/openai/gpt-oss-20b"], + concurrencySemaphore: sem, + queueTimeoutMs: 20, + onModelSwitch, + }); + + expect(result).toBe("cloud-fallback-result"); + expect(stats.modelUsed).toBe("groq/openai/gpt-oss-20b"); + expect(onModelSwitch).toHaveBeenCalledWith( + "openai/llama3", + "groq/openai/gpt-oss-20b", + expect.any(VRAMConcurrencyError), + 1, + ); + + blockRelease(); + }); + + it("immediately re-throws and does not retry when request signal is aborted", async () => { + const controller = new AbortController(); + controller.abort(); + + const op = vi.fn(async () => "ok"); + + await expect( + executeWithFallbackAndRetry(op, { + primaryModel: "primary", + fallbackModels: ["secondary"], + signal: controller.signal, + }), + ).rejects.toThrow(); + + expect(op).not.toHaveBeenCalled(); + }); + }); }); diff --git a/src/ai/resilience.ts b/src/ai/resilience.ts index f38978f3..99f8a7b8 100644 --- a/src/ai/resilience.ts +++ b/src/ai/resilience.ts @@ -133,12 +133,264 @@ export interface RetryConfig { timeoutMs?: number; } +// --------------------------------------------------------------------------- +// Semaphore & VRAM Concurrency Control (#1753) +// --------------------------------------------------------------------------- + +export interface SemaphoreAcquireOptions { + timeoutMs?: number; + signal?: AbortSignal; +} + +/** + * Concurrency semaphore with FIFO queueing, timeout bounds, and abort support (#1753). + * Limits simultaneous active GPU/VRAM generations to prevent local inference crashes. + */ +export class Semaphore { + private permits: number; + private maxPermits: number; + private queue: Array<{ + resolve: (release: () => void) => void; + reject: (err: Error) => void; + timer?: NodeJS.Timeout; + signal?: AbortSignal; + onAbort?: () => void; + }> = []; + + constructor(permits = 1) { + this.maxPermits = Math.max(1, permits); + this.permits = this.maxPermits; + } + + get availablePermits(): number { + return this.permits; + } + + get queueLength(): number { + return this.queue.length; + } + + get capacity(): number { + return this.maxPermits; + } + + setCapacity(newCapacity: number): void { + const diff = newCapacity - this.maxPermits; + this.maxPermits = Math.max(1, newCapacity); + this.permits += diff; + while (this.permits > 0 && this.queue.length > 0) { + this.dispatchNext(); + } + } + + async acquire(options?: SemaphoreAcquireOptions): Promise<() => void> { + if (options?.signal?.aborted) { + const err = new Error("Operation aborted while acquiring semaphore permit"); + err.name = "AbortError"; + throw err; + } + + if (this.permits > 0) { + this.permits--; + let released = false; + return () => { + if (!released) { + released = true; + this.release(); + } + }; + } + + return new Promise<() => void>((resolve, reject) => { + let timer: NodeJS.Timeout | undefined; + let onAbort: (() => void) | undefined; + + const entry = { + resolve: (releaseFn: () => void) => { + if (timer) clearTimeout(timer); + if (options?.signal && onAbort) { + options.signal.removeEventListener("abort", onAbort); + } + resolve(releaseFn); + }, + reject: (err: Error) => { + if (timer) clearTimeout(timer); + if (options?.signal && onAbort) { + options.signal.removeEventListener("abort", onAbort); + } + reject(err); + }, + }; + + if (options?.timeoutMs && options.timeoutMs > 0) { + timer = setTimeout(() => { + const idx = this.queue.indexOf(entry); + if (idx !== -1) { + this.queue.splice(idx, 1); + entry.reject( + new VRAMConcurrencyError( + `Local model VRAM concurrency queue wait timed out after ${options.timeoutMs}ms`, + ), + ); + } + }, options.timeoutMs); + } + + if (options?.signal) { + onAbort = () => { + const idx = this.queue.indexOf(entry); + if (idx !== -1) { + this.queue.splice(idx, 1); + const err = new Error("Operation aborted while waiting in semaphore queue"); + err.name = "AbortError"; + entry.reject(err); + } + }; + options.signal.addEventListener("abort", onAbort, { once: true }); + } + + this.queue.push(entry); + }); + } + + private dispatchNext(): void { + if (this.queue.length > 0 && this.permits > 0) { + this.permits--; + const next = this.queue.shift(); + if (next) { + let released = false; + next.resolve(() => { + if (!released) { + released = true; + this.release(); + } + }); + } + } + } + + private release(): void { + this.permits++; + this.dispatchNext(); + } + + reset(): void { + while (this.queue.length > 0) { + const item = this.queue.shift(); + const err = new Error("Semaphore reset"); + item?.reject(err); + } + this.permits = this.maxPermits; + } +} + +/** + * Thrown when local inference requests exceed available GPU memory / concurrency limits (#1753). + */ +export class VRAMConcurrencyError extends Error { + constructor(message = "Local model VRAM concurrency threshold exceeded") { + super(message); + this.name = "VRAMConcurrencyError"; + } +} + +/** + * Identifies errors caused by saturated local model VRAM or semaphore concurrency queue exhaustion. + */ +export function isVramConcurrencyError(err: unknown): boolean { + if (!err) return false; + if (err instanceof VRAMConcurrencyError) return true; + const msg = err instanceof Error ? err.message : String(err); + return ( + /vram (?:concurrency|threshold)/i.test(msg) || + /concurrency queue/i.test(msg) || + /exceeded vram/i.test(msg) + ); +} + +/** + * Checks whether a given model identifier represents a locally-hosted LLM (e.g. Ollama, vLLM). + */ +export function isLocalModel(modelName: string): boolean { + if (!modelName) return false; + const name = String(modelName).toLowerCase(); + return ( + name.startsWith("openai/") || + name.startsWith("local/") || + name.includes("localhost") || + name.includes("ollama") || + name.includes("vllm") + ); +} + +export const DEFAULT_LOCAL_VRAM_CONCURRENCY = 1; + +/** + * Resolves maximum permitted concurrent local model executions from environment. + */ +export function getLocalModelVramThreshold( + env: Readonly> = process.env, +): number { + const raw = + env.LOCAL_AI_MAX_CONCURRENCY ?? + env.LOCAL_AI_CONCURRENCY_LIMIT ?? + env.VRAM_CONCURRENCY_THRESHOLD; + const parsed = Number(raw); + return Number.isFinite(parsed) && parsed > 0 ? parsed : DEFAULT_LOCAL_VRAM_CONCURRENCY; +} + +let localModelSemaphore: Semaphore | null = null; + +export function getLocalModelSemaphore(capacity?: number): Semaphore { + if (!localModelSemaphore) { + const limit = capacity ?? getLocalModelVramThreshold(); + localModelSemaphore = new Semaphore(limit); + } + return localModelSemaphore; +} + +export function resetLocalModelSemaphore(): void { + if (localModelSemaphore) { + localModelSemaphore.reset(); + } + localModelSemaphore = null; +} + +/** + * Executes an operation within a concurrency semaphore slot. + */ +export async function withModelConcurrency( + fn: () => Promise, + options?: { semaphore?: Semaphore; timeoutMs?: number; signal?: AbortSignal }, +): Promise { + const sem = options?.semaphore ?? getLocalModelSemaphore(); + const release = await sem.acquire({ + timeoutMs: options?.timeoutMs, + signal: options?.signal, + }); + try { + return await fn(); + } finally { + release(); + } +} + export interface ModelFallbackConfig { primaryModel: TModel; fallbackModels: TModel[]; retryConfig?: RetryConfig; circuitBreakerOptions?: CircuitBreakerOptions; - onModelSwitch?: (fromModel: TModel, toModel: TModel, error: unknown, attempt: number) => void; + concurrencySemaphore?: Semaphore; + semaphore?: Semaphore; + queueTimeoutMs?: number; + signal?: AbortSignal; + onModelSwitch?: ( + fromModel: TModel, + toModel: TModel, + error: unknown, + attempt: number, + reasonCode?: string, + ) => void; } export interface ResilienceExecutionStats { @@ -234,6 +486,12 @@ export async function executeWithFallbackAndRetry {}); if (config.onModelSwitch) { config.onModelSwitch(currentModel, nextModel, openError, 0); @@ -267,11 +526,24 @@ export async function executeWithFallbackAndRetry void) | undefined; try { + if (sem) { + release = await sem.acquire({ + timeoutMs: config.queueTimeoutMs ?? 5000, + signal: config.signal, + }); + } + const timeoutMs = config.retryConfig?.timeoutMs ?? DEFAULT_ATTEMPT_TIMEOUT_MS; const result = await breaker.execute(() => withTimeout(() => operation(currentModel, attempt), timeoutMs), @@ -304,14 +576,20 @@ export async function executeWithFallbackAndRetry {}); + if (config.signal?.aborted || (err as any)?.name === "AbortError") { + throw err; + } + + const isVram = isVramConcurrencyError(err); const isBreakerOpen = err instanceof CircuitBreakerError; const isRetryable = + !isVram && !isBreakerOpen && (isRateLimitError(err) || isTimeoutError(err) || Boolean( config.retryConfig?.retryableErrors && - config.retryConfig.retryableErrors.some((fn) => fn(err)), + config.retryConfig.retryableErrors.some((fn) => fn(err)), )); if (attempt < maxRetries && isRetryable && breaker.getState() !== CircuitState.OPEN) { @@ -338,8 +616,18 @@ export async function executeWithFallbackAndRetry {}); if (config.onModelSwitch) { @@ -356,6 +645,8 @@ export async function executeWithFallbackAndRetry { } }); }); + describe("Failover Reason Codes (#1753)", () => { + it("records explicit reasonCode in failover event", async () => { + await recordFailoverEvent({ + fromModel: "openai/llama3", + toModel: "groq/openai/gpt-oss-20b", + error: new Error("Local model VRAM concurrency threshold exceeded"), + attempt: 1, + reasonCode: "vram_concurrency_exceeded", + }); + + const report = await getAiHealthReport(); + const event = report.recentFailovers[0]; + expect(event).toBeDefined(); + expect(event.reasonCode).toBe("vram_concurrency_exceeded"); + expect(event.fromModel).toBe("openai/llama3"); + expect(event.toModel).toBe("groq/openai/gpt-oss-20b"); + }); + + it("auto-detects vram_concurrency_exceeded from error message when reasonCode omitted", async () => { + await recordFailoverEvent({ + fromModel: "openai/llama3", + toModel: "groq/openai/gpt-oss-20b", + error: new Error("VRAM concurrency semaphore capacity reached"), + attempt: 1, + }); + + const report = await getAiHealthReport(); + const event = report.recentFailovers[0]; + expect(event).toBeDefined(); + expect(event.reasonCode).toBe("vram_concurrency_exceeded"); + }); + + it("auto-detects rate_limit and timeout reason codes", async () => { + await recordFailoverEvent({ + fromModel: "groq/primary", + toModel: "groq/fallback", + error: new Error("Rate limit exceeded 429"), + attempt: 1, + }); + + const report = await getAiHealthReport(); + const event = report.recentFailovers[0]; + expect(event.reasonCode).toBe("rate_limit"); + }); + }); }); diff --git a/src/ai/telemetry.ts b/src/ai/telemetry.ts index 5e1aea51..eb1b1c75 100644 --- a/src/ai/telemetry.ts +++ b/src/ai/telemetry.ts @@ -30,6 +30,16 @@ export interface LatencySample { error?: string; } +export type FallbackReasonCode = + | "vram_concurrency_exceeded" + | "concurrency_limit" + | "rate_limit" + | "timeout" + | "insufficient_memory" + | "circuit_breaker_open" + | "model_exhausted" + | "unknown"; + export interface FailoverEvent { id: string; timestamp: number; @@ -38,6 +48,8 @@ export interface FailoverEvent { error: string; attempt: number; fastFail: boolean; + reasonCode?: string; + reason?: string; } export interface ModelHealthStatus { @@ -215,19 +227,52 @@ export async function recordFailoverEvent(event: { error: unknown; attempt: number; fastFail?: boolean; + reasonCode?: string; + reason?: string; }): Promise { + const errorMsg = event.error + ? event.error instanceof Error + ? event.error.message + : String(event.error) + : "Unknown error"; + + let detectedReasonCode = event.reasonCode || event.reason; + if (!detectedReasonCode) { + if (event.fastFail) { + detectedReasonCode = "circuit_breaker_open"; + } else { + const lower = errorMsg.toLowerCase(); + if (lower.includes("vram") || lower.includes("concurrency") || lower.includes("semaphore")) { + detectedReasonCode = "vram_concurrency_exceeded"; + } else if (lower.includes("rate limit") || lower.includes("429") || lower.includes("quota")) { + detectedReasonCode = "rate_limit"; + } else if ( + lower.includes("timeout") || + lower.includes("timed out") || + lower.includes("503") || + lower.includes("504") + ) { + detectedReasonCode = "timeout"; + } else if (lower.includes("memory") || lower.includes("oom")) { + detectedReasonCode = "insufficient_memory"; + } else if (lower.includes("circuit")) { + detectedReasonCode = "circuit_breaker_open"; + } else { + detectedReasonCode = "unknown"; + } + } + } + const fullEvent: FailoverEvent = { id: `failover-${Date.now()}-${Math.random().toString(36).slice(2, 7)}`, timestamp: Date.now(), fromModel: normalizeModelName(event.fromModel), toModel: normalizeModelName(event.toModel), - error: event.error - ? event.error instanceof Error - ? event.error.message - : String(event.error) - : "Unknown error", + error: errorMsg, attempt: event.attempt, fastFail: Boolean(event.fastFail), + reasonCode: detectedReasonCode, + reason: event.reason || detectedReasonCode, }; memoryStore.registeredModels.add(fullEvent.fromModel); diff --git a/src/app/api/findings/[id]/explain-stream/route.test.ts b/src/app/api/findings/[id]/explain-stream/route.test.ts index d39301f4..b464086e 100644 --- a/src/app/api/findings/[id]/explain-stream/route.test.ts +++ b/src/app/api/findings/[id]/explain-stream/route.test.ts @@ -43,6 +43,10 @@ vi.mock("@/lib/redis", () => ({ redis: null, })); +vi.mock("@/lib/rate-limit", () => ({ + ratelimit: null, +})); + const mockFinding = { id: "finding-1", type: "Vulnerability", diff --git a/src/app/api/findings/[id]/explain-stream/route.ts b/src/app/api/findings/[id]/explain-stream/route.ts index e53c7526..4c430c03 100644 --- a/src/app/api/findings/[id]/explain-stream/route.ts +++ b/src/app/api/findings/[id]/explain-stream/route.ts @@ -118,22 +118,31 @@ async function handler(request: NextRequest, { params }: { params: Promise<{ id: }; const finish = (): void => { - // Unconditional, and before the `closed` guard. `closed` tracks - // whether the controller may still be written to -- `send` sets it on - // a failed enqueue -- which is a different question from whether the - // registry still holds this connection. Gating both on the one flag - // meant a failed enqueue made every later finish() a no-op and the - // entry was never released (#722). - release(); - - if (closed) return; + if (closed) { + release(); + return; + } closed = true; try { controller.close(); } catch {} + release(); }; - if (abortSignal.aborted) { + const onClientAbort = () => { + closed = true; + finish(); + }; + + if (request.signal) { + if (request.signal.aborted) { + onClientAbort(); + return; + } + request.signal.addEventListener("abort", onClientAbort, { once: true }); + } + + if (abortSignal.aborted || request.signal?.aborted) { finish(); return; } @@ -152,7 +161,7 @@ async function handler(request: NextRequest, { params }: { params: Promise<{ id: }, { signal: abortSignal }, )) { - if (closed || abortSignal.aborted) { + if (closed || abortSignal.aborted || request.signal?.aborted) { finish(); return; }