Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/local-model-vram-abort-1753.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"secureflow": patch
---

fix(ai): wire request abort signals into local model generation and enforce semaphore-based VRAM concurrency queue (#1753)
27 changes: 25 additions & 2 deletions src/ai/flows/security-explanation-core.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { InsufficientVRAMError, isInsufficientMemoryError } from "@/ai/local-model";
import { getModelId } from "@/lib/explanation-cache";
import { isRateLimitError, isTimeoutError } from "./security-helpers";
import { getModelCircuitBreaker, CircuitBreakerError, CircuitState } from "@/ai/resilience";
import { getModelCircuitBreaker, CircuitBreakerError, CircuitState, isVramConcurrencyError } from "@/ai/resilience";
import {
AISecurityExplanationOutputSchema,
type AISecurityExplanationInput,
Expand All @@ -18,7 +18,11 @@
}

export type FallbackReason =
"rate_limit" | "timeout" | "insufficient_memory" | "circuit_breaker_open";
| "rate_limit"
| "timeout"
| "insufficient_memory"
| "circuit_breaker_open"
| "vram_concurrency_exceeded";

export function prepareExplanation(input: AISecurityExplanationInput) {
return {
Expand Down Expand Up @@ -82,6 +86,25 @@
continue;
}

if (isVramConcurrencyError(error)) {
if (next === undefined) {
throw error;
}

console.warn(
`[AI_FALLBACK] Model ${String(model)} exceeded VRAM concurrency limit. Switching to ${String(next)}`,
);

onFallback?.({
from: model,
to: next,
reason: "vram_concurrency_exceeded",
error,
});

continue;
}

if (isInsufficientMemoryError(error)) {
if (next === undefined) {
throw new InsufficientVRAMError(models.map((m) => getModelId(m as any)));
Expand Down
120 changes: 120 additions & 0 deletions src/ai/local-model.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ import {
DEFAULT_VLLM_MODEL,
DEFAULT_LOCAL_AI_URL,
DEFAULT_LOCAL_AI_MODEL,
streamLocalModelGeneration,
generateLocalModel,
} from "./local-model";

// ---------------------------------------------------------------------------
Expand Down Expand Up @@ -434,3 +436,121 @@ describe("default constants", () => {
expect(DEFAULT_VLLM_MODEL).toBe("meta-llama/Meta-Llama-3-8B-Instruct");
});
});


// ---------------------------------------------------------------------------
// streamLocalModelGeneration & Abort Signal Wire-up (#1753)
// ---------------------------------------------------------------------------

describe("streamLocalModelGeneration & generateLocalModel (#1753)", () => {
it("returns immediately without fetching if signal is already aborted", async () => {
const controller = new AbortController();
controller.abort();

const fetchSpy = vi.spyOn(globalThis, "fetch");

const tokens: string[] = [];
for await (const chunk of streamLocalModelGeneration({
prompt: "test",
signal: controller.signal,
})) {
tokens.push(chunk);
}

expect(tokens).toEqual([]);
expect(fetchSpy).not.toHaveBeenCalled();
});

it("streams tokens from OpenAI-compatible SSE chunks", async () => {
const sseData =
'data: {"choices":[{"delta":{"content":"Hello"}}]}\n\n' +
'data: {"choices":[{"delta":{"content":" world"}}]}\n\n' +
'data: [DONE]\n\n';

const encoder = new TextEncoder();
const stream = new ReadableStream({
start(ctrl) {
ctrl.enqueue(encoder.encode(sseData));
ctrl.close();
},
});

vi.spyOn(globalThis, "fetch").mockResolvedValueOnce({
ok: true,
status: 200,
body: stream,
} as any);

const tokens: string[] = [];
for await (const token of streamLocalModelGeneration({
prompt: "hi",
config: { baseUrl: "http://localhost:11434/v1", model: "llama3" },
})) {
tokens.push(token);
}

expect(tokens).toEqual(["Hello", " world"]);
});

it("halts token generation and cancels reader immediately on client abort signal", async () => {
const controller = new AbortController();
let readerCancelled = false;

const encoder = new TextEncoder();
const stream = new ReadableStream({
start(ctrl) {
ctrl.enqueue(encoder.encode('data: {"choices":[{"delta":{"content":"Token 1"}}]}\n\n'));
},
cancel() {
readerCancelled = true;
},
});

vi.spyOn(globalThis, "fetch").mockResolvedValueOnce({
ok: true,
status: 200,
body: stream,
} as any);

const tokens: string[] = [];
for await (const token of streamLocalModelGeneration({
prompt: "abort test",
signal: controller.signal,
config: { baseUrl: "http://localhost:11434/v1", model: "llama3" },
})) {
tokens.push(token);
controller.abort();
}

expect(tokens).toEqual(["Token 1"]);
expect(readerCancelled).toBe(true);
});

it("generateLocalModel accumulates all streamed tokens into complete text", async () => {
const sseData =
'data: {"choices":[{"delta":{"content":"Secure"}}]}\n\n' +
'data: {"choices":[{"delta":{"content":"Flow"}}]}\n\n' +
'data: [DONE]\n\n';

const encoder = new TextEncoder();
const stream = new ReadableStream({
start(ctrl) {
ctrl.enqueue(encoder.encode(sseData));
ctrl.close();
},
});

vi.spyOn(globalThis, "fetch").mockResolvedValueOnce({
ok: true,
status: 200,
body: stream,
} as any);

const result = await generateLocalModel({
prompt: "test accumulation",
config: { baseUrl: "http://localhost:11434/v1", model: "llama3" },
});

expect(result.text).toBe("SecureFlow");
});
});
152 changes: 152 additions & 0 deletions src/ai/local-model.ts
Original file line number Diff line number Diff line change
Expand Up @@ -364,6 +364,158 @@ export class InsufficientVRAMError extends Error {
}
}


// ---------------------------------------------------------------------------
// Local Model Generation & Streaming Loops with Abort Support (#1753)
// ---------------------------------------------------------------------------

export interface LocalGenerationOptions {
prompt: string;
system?: string;
model?: string;
config?: LocalModelConfig;
signal?: AbortSignal;
temperature?: number;
maxTokens?: number;
}

/**
* Streams tokens from a local OpenAI-compatible inference server (Ollama, vLLM).
* Wires the caller's AbortSignal directly into the fetch call and reader loop
* so client disconnects halt GPU token generation immediately without pinning VRAM (#1753).
*/
export async function* streamLocalModelGeneration(
options: LocalGenerationOptions,
): AsyncGenerator<string, void, unknown> {
const signal = options.signal;
if (signal?.aborted) return;

const config =
options.config ||
resolveLocalModelConfig() || {
baseUrl: DEFAULT_LOCAL_AI_URL,
model: options.model || DEFAULT_LOCAL_AI_MODEL,
provider: "ollama",
apiKey: "local",
};

const endpoint = `${config.baseUrl}/chat/completions`;
const body = {
model: options.model || config.model,
messages: [
...(options.system ? [{ role: "system", content: options.system }] : []),
{ role: "user", content: options.prompt },
],
stream: true,
temperature: options.temperature ?? 0.1,
max_tokens: options.maxTokens ?? 1024,
};

let response: Response;
try {
response = await fetch(endpoint, {
method: "POST",
headers: {
"Content-Type": "application/json",
Authorization: `Bearer ${config.apiKey || "local"}`,
},
body: JSON.stringify(body),
signal,
});
} catch (err: any) {
if (signal?.aborted || err?.name === "AbortError") {
return;
}
throw err;
}

if (!response.ok) {
const errorText = await response.text().catch(() => "");
throw new Error(`Local model request failed with status ${response.status}: ${errorText}`);
}

const reader = response.body?.getReader();
if (!reader) return;

const decoder = new TextDecoder();
let buffer = "";

const onAbort = () => {
reader.cancel().catch(() => {});
};
if (signal) {
signal.addEventListener("abort", onAbort, { once: true });
}

try {
while (true) {
if (signal?.aborted) {
await reader.cancel().catch(() => {});
return;
}

let readResult: ReadableStreamReadResult<Uint8Array>;
try {
readResult = await reader.read();
} catch (readErr: any) {
if (signal?.aborted || readErr?.name === "AbortError") {
return;
}
throw readErr;
}

const { done, value } = readResult;
if (done) break;

buffer += decoder.decode(value, { stream: true });
const lines = buffer.split("\n");
buffer = lines.pop() ?? "";

for (const line of lines) {
if (signal?.aborted) {
await reader.cancel().catch(() => {});
return;
}

const trimmed = line.trim();
if (!trimmed || trimmed.startsWith(":")) continue;
if (trimmed === "data: [DONE]") return;
if (trimmed.startsWith("data: ")) {
try {
const data = JSON.parse(trimmed.slice(6));
const content = data.choices?.[0]?.delta?.content ?? "";
if (content) {
yield content;
}
} catch {
// Ignore unparseable partial chunk
}
}
}
}
} finally {
if (signal && onAbort) {
signal.removeEventListener("abort", onAbort);
}
try {
await reader.cancel();
} catch {}
}
}

/**
* Runs a local model completion with direct abort signal support (#1753).
*/
export async function generateLocalModel(
options: LocalGenerationOptions,
): Promise<{ text: string }> {
let fullText = "";
for await (const chunk of streamLocalModelGeneration(options)) {
fullText += chunk;
}
return { text: fullText };
}

// ── ArmorIQ Pre-Execution Intent Signing (#1275) ──────────────────────────
export {
declareAndSignIntent as emitExecutionPlan,
Expand Down
Loading
Loading