From c87321a3d30afd33d83d96f4fdec75b0de4ce403 Mon Sep 17 00:00:00 2001 From: ollypee22 Date: Wed, 30 Sep 2026 05:52:07 +0000 Subject: [PATCH] feat(scenarios): run seed catalog through the decision engine and fail on drift (#216) The scenario catalog, the seed data, and the decision engine could change separately, so a scenario card could still show "deny" while the engine allows - a broken demo of the firewall. - Add src/lib/scenarios/evaluate.ts: a deterministic engine harness for the seeded catalog (default policy minus the allowed-hours window, seeded usage snapshot) plus validateScenarioPack(), which reports missing expected decisions and seed-vs-engine drift with engine rule ids in the message. - Extend scenario-pack.test.ts with a drift guard: every seed entry is evaluated through the engine, a mutated expected decision fails the suite, a seed entry without an expected decision fails the suite, and the suite proves it never contacts Horizon or any other network (fetch is stubbed to reject and FORTEXA_BLOCKLIST_URL is cleared). - The scenarios page renders engine results instead of hard-coded labels: settings/page.tsx evaluates the catalog server-side and passes the evaluations to ScenariosCatalog; badges and the decision filter use the engine decision, and each card gains an "Engine decision" row. - scripts/demo-scenarios.ts reuses the harness and exits non-zero when any seed expectation drifts from engine output. - Repair pre-existing parse errors that blocked the suite: restore the mangled fetchBlocklistWithTimeout in src/lib/security/analyzer.ts (timeout and degradation semantics per analyzer.test.ts), restore a stripped useCallback wrapper in src/lib/auth/use-auth-session.ts, and remove a duplicated const left behind in src/app/api/audit/export/route.test.ts. --- scripts/demo-scenarios.ts | 62 ++++++-- src/app/api/audit/export/route.test.ts | 2 - src/app/settings/page.tsx | 7 +- src/components/scenarios-catalog.test.tsx | 115 +++++++++++++-- src/components/scenarios-catalog.tsx | 35 ++++- src/lib/auth/use-auth-session.ts | 4 +- src/lib/scenarios/evaluate.ts | 165 ++++++++++++++++++++++ src/lib/scenarios/scenario-pack.test.ts | 88 +++++++++++- src/lib/security/analyzer.ts | 74 +++++----- 9 files changed, 481 insertions(+), 71 deletions(-) create mode 100644 src/lib/scenarios/evaluate.ts diff --git a/scripts/demo-scenarios.ts b/scripts/demo-scenarios.ts index 514f6ff7..e49bb60b 100644 --- a/scripts/demo-scenarios.ts +++ b/scripts/demo-scenarios.ts @@ -1,22 +1,58 @@ -import { evaluateDecision } from "../src/lib/decision/engine"; -import { defaultPolicyConfig } from "../src/lib/policy/engine"; -import { demoScenarios, defaultDailyUsage } from "../src/lib/scenarios/seed"; +import { + evaluateScenarioCatalog, + validateScenarioPack, +} from "../src/lib/scenarios/evaluate"; +import { demoScenarios } from "../src/lib/scenarios/seed"; (async () => { console.log("Fortexa Scenario Demo Runner\n"); - for (const scenario of demoScenarios) { - const result = await evaluateDecision(scenario.action, defaultPolicyConfig, defaultDailyUsage); - console.log(`Scenario: ${scenario.title}`); - console.log(`Expected: ${scenario.expectedDecision} | Actual: ${result.decision}`); - console.log(`Risk Score: ${result.riskScore}`); - console.log(`Explanation: ${result.explanation}`); - if (result.triggeredPolicies.length > 0) { - console.log(`Policies: ${result.triggeredPolicies.map((item) => item.code).join(", ")}`); + const evaluations = await evaluateScenarioCatalog(); + let driftCount = 0; + + for (const evaluation of evaluations) { + console.log(`Scenario: ${evaluation.scenario.title}`); + console.log( + `Expected: ${evaluation.expected} | Actual: ${evaluation.actual}`, + ); + console.log(`Risk Score: ${evaluation.result.riskScore}`); + console.log(`Explanation: ${evaluation.result.explanation}`); + if (evaluation.result.triggeredPolicies.length > 0) { + console.log( + `Policies: ${evaluation.result.triggeredPolicies + .map((item) => item.code) + .join(", ")}`, + ); + } + if (evaluation.result.riskFindings.length > 0) { + console.log( + `Findings: ${evaluation.result.riskFindings + .map((item) => item.code) + .join(", ")}`, + ); } - if (result.riskFindings.length > 0) { - console.log(`Findings: ${result.riskFindings.map((item) => item.code).join(", ")}`); + if (!evaluation.matches) { + driftCount += 1; + console.log(`DRIFT: ${evaluation.scenario.id} no longer matches its seed.`); } console.log("---"); } + + const issues = validateScenarioPack({ + scenarios: demoScenarios, + evaluations, + }); + for (const issue of issues) { + console.error(`DRIFT: ${issue.message}`); + } + + if (driftCount > 0 || issues.length > 0) { + console.error( + `\n${Math.max(driftCount, issues.length)} seeded scenario(s) drifted from engine output. Failing.`, + ); + process.exitCode = 1; + return; + } + + console.log("\nAll seeded scenarios match engine output."); })(); diff --git a/src/app/api/audit/export/route.test.ts b/src/app/api/audit/export/route.test.ts index 936c5abf..2aa86406 100644 --- a/src/app/api/audit/export/route.test.ts +++ b/src/app/api/audit/export/route.test.ts @@ -172,8 +172,6 @@ describe("/api/audit/export route", () => { }, } ); - const response = await GET(request); - const response = await GET(req); expect(response.status).toBe(200); diff --git a/src/app/settings/page.tsx b/src/app/settings/page.tsx index 89798fc2..9ebf6a62 100644 --- a/src/app/settings/page.tsx +++ b/src/app/settings/page.tsx @@ -4,6 +4,7 @@ import { ActivityTimeline } from "@/components/activity-timeline"; import { OpsDashboard } from "@/components/ops-dashboard"; import { PolicyEditor } from "@/components/policy-editor"; import { ScenariosCatalog } from "@/components/scenarios-catalog"; +import { evaluateScenarioCatalog } from "@/lib/scenarios/evaluate"; import { TabNav, type TabItem } from "@/components/ui/tab-nav"; import { WalletStatusCard } from "@/components/wallet-status-card"; import { AUTH_COOKIE_KEY, verifySessionToken } from "@/lib/auth/session"; @@ -30,6 +31,8 @@ export default async function SettingsPage({ const session = sessionToken ? verifySessionToken(sessionToken) : null; const userId = session?.userId; const entries = userId ? await listAuditEntries(userId) : []; + const scenarioEvaluations = + activeTab === "scenarios" ? await evaluateScenarioCatalog() : []; return (
@@ -57,7 +60,9 @@ export default async function SettingsPage({
) : null} - {activeTab === "scenarios" ? : null} + {activeTab === "scenarios" ? ( + + ) : null} {activeTab === "ops" ? : null} {activeTab === "activity" ? : null} diff --git a/src/components/scenarios-catalog.test.tsx b/src/components/scenarios-catalog.test.tsx index 2e359a0e..f59c0387 100644 --- a/src/components/scenarios-catalog.test.tsx +++ b/src/components/scenarios-catalog.test.tsx @@ -45,7 +45,43 @@ vi.mock("@/components/ui/button", () => ({ })); import { ScenariosCatalog } from "./scenarios-catalog"; +import type { ScenarioEvaluation } from "@/lib/scenarios/evaluate"; import { demoScenarios } from "@/lib/scenarios/seed"; +import type { DecisionResult, DecisionType } from "@/lib/types/domain"; + +function makeEvaluation( + scenario: (typeof demoScenarios)[number], + actualDecision: DecisionType, +): ScenarioEvaluation { + const result: DecisionResult = { + decision: actualDecision, + explanation: `Engine says ${actualDecision}`, + triggeredPolicies: [], + riskScore: 10, + riskFindings: [], + requiresManualApproval: actualDecision === "REQUIRE_APPROVAL", + analyzerStatus: { + blocklistStatus: "success", + isDegraded: false, + degradationReasons: [], + }, + }; + + return { + scenario, + expected: scenario.expectedDecision, + actual: result.decision, + result, + matches: result.decision === scenario.expectedDecision, + }; +} + +const baseEvaluations: ScenarioEvaluation[] = demoScenarios.map((scenario) => + makeEvaluation(scenario, scenario.expectedDecision) +); + +const renderCatalog = (evaluations: ScenarioEvaluation[] = baseEvaluations) => + ScenariosCatalog({ evaluations }); describe("ScenariosCatalog Search, Filter, and Empty State", () => { beforeEach(() => { @@ -61,7 +97,7 @@ describe("ScenariosCatalog Search, Filter, and Empty State", () => { it("initially shows all scenarios and has empty search and ALL decision filter", () => { hookIndex = 0; - const element = ScenariosCatalog(); + const element = renderCatalog(); expect(getSearchQuery()).toBe(""); expect(getSelectedDecision()).toBe("ALL"); @@ -76,14 +112,14 @@ describe("ScenariosCatalog Search, Filter, and Empty State", () => { it("filters scenarios based on search query", () => { // Render first time hookIndex = 0; - ScenariosCatalog(); + renderCatalog(); // Set search query to match "Suspicious" setSearchQuery("suspicious"); // Render second time with new state hookIndex = 0; - const element = ScenariosCatalog(); + const element = renderCatalog(); const gridDiv = element.props.children[1]; expect(gridDiv.type).toBe("div"); @@ -101,14 +137,14 @@ describe("ScenariosCatalog Search, Filter, and Empty State", () => { it("filters scenarios based on decision badge", () => { // Render first time hookIndex = 0; - ScenariosCatalog(); + renderCatalog(); // Set filter to "BLOCK" setSelectedDecision("BLOCK"); // Render second time with new state hookIndex = 0; - const element = ScenariosCatalog(); + const element = renderCatalog(); const gridDiv = element.props.children[1]; expect(gridDiv.type).toBe("div"); @@ -117,17 +153,76 @@ describe("ScenariosCatalog Search, Filter, and Empty State", () => { expect(gridDiv.props.children.length).toBe(expectedCount); }); + it("applies the decision filter to engine decisions, not seed labels", () => { + hookIndex = 0; + const fixture = demoScenarios.find((s) => s.id === "safe-research-payment")!; + const driftedEvaluations = [ + makeEvaluation(fixture, "BLOCK"), + ...demoScenarios + .filter((s) => s.id !== fixture.id) + .map((scenario) => makeEvaluation(scenario, scenario.expectedDecision)), + ]; + + renderCatalog(driftedEvaluations); + + // Seed label is APPROVE; the engine decided BLOCK. + setSelectedDecision("BLOCK"); + + hookIndex = 0; + const element = renderCatalog(driftedEvaluations); + + const gridDiv = element.props.children[1]; + const renderedIds = gridDiv.props.children.map( + (child: { props: { "data-scenario-id": string } }) => child.props["data-scenario-id"] + ); + // The drifted fixture is included with its engine decision, plus every seed that + // genuinely expects BLOCK (including blocked-malicious-endpoint). + const seededBlockCount = demoScenarios.filter((s) => s.expectedDecision === "BLOCK").length; + expect(renderedIds).toContain(fixture.id); + expect(renderedIds).toHaveLength(seededBlockCount + 1); + }); + + it("renders the engine decision for a fixture scenario instead of the seed label", () => { + hookIndex = 0; + const fixture = demoScenarios.find((s) => s.id === "safe-research-payment")!; + const driftedEvaluations = [ + makeEvaluation(fixture, "REQUIRE_APPROVAL"), + ...demoScenarios + .filter((s) => s.id !== fixture.id) + .map((scenario) => makeEvaluation(scenario, scenario.expectedDecision)), + ]; + + const element = renderCatalog(driftedEvaluations); + + const gridDiv = element.props.children[1]; + const card = gridDiv.props.children.find( + (child: { props: { "data-scenario-id": string } }) => + child.props["data-scenario-id"] === fixture.id + ); + expect(card).toBeDefined(); + + // Seed label says APPROVE; the engine result (REQUIRE_APPROVAL) must win. + const badge = card.props.children[0].props.children[1]; + expect(badge.props.decision).toBe("REQUIRE_APPROVAL"); + + const engineDecisionRow = card.props.children[2].props.children[3]; + expect(engineDecisionRow.props.children[0].props.children).toBe("Engine decision"); + const engineDecisionValue = engineDecisionRow.props.children[1]; + expect(engineDecisionValue.props["data-engine-decision"]).toBe(fixture.id); + expect(engineDecisionValue.props.children).toBe("REQUIRE_APPROVAL"); + }); + it("displays empty state when no scenarios match filters", () => { // Render first time hookIndex = 0; - ScenariosCatalog(); + renderCatalog(); // Set search query to non-existent value setSearchQuery("nonexistent_search_query_that_matches_nothing_12345"); // Render second time hookIndex = 0; - const element = ScenariosCatalog(); + const element = renderCatalog(); // The second child should be the empty state container const emptyStateDiv = element.props.children[1]; @@ -148,7 +243,7 @@ describe("ScenariosCatalog Search, Filter, and Empty State", () => { it("restores scenario list immediately when clear filters is clicked", () => { // Render first time hookIndex = 0; - ScenariosCatalog(); + renderCatalog(); // Set search query to non-existent value to trigger empty state setSearchQuery("nonexistent_search_query_that_matches_nothing_12345"); @@ -156,7 +251,7 @@ describe("ScenariosCatalog Search, Filter, and Empty State", () => { // Verify empty state is displayed hookIndex = 0; - let element = ScenariosCatalog(); + let element = renderCatalog(); expect(element.props.children[1].props.id).toBe("scenario-empty-state"); // Click the clear filters button (directly invoke the onClick callback) @@ -165,7 +260,7 @@ describe("ScenariosCatalog Search, Filter, and Empty State", () => { // Render again hookIndex = 0; - element = ScenariosCatalog(); + element = renderCatalog(); // Should be back to initial state expect(getSearchQuery()).toBe(""); diff --git a/src/components/scenarios-catalog.tsx b/src/components/scenarios-catalog.tsx index 84c9c949..12a3ab40 100644 --- a/src/components/scenarios-catalog.tsx +++ b/src/components/scenarios-catalog.tsx @@ -2,15 +2,30 @@ import { useState, useMemo } from "react"; import { DecisionBadge } from "@/components/decision-badge"; +import type { ScenarioEvaluation } from "@/lib/scenarios/evaluate"; import { demoScenarios } from "@/lib/scenarios/seed"; +import type { Scenario } from "@/lib/types/domain"; import { Input } from "@/components/ui/input"; import { Button } from "@/components/ui/button"; -export function ScenariosCatalog() { +export function ScenariosCatalog({ + evaluations, +}: { + evaluations: ScenarioEvaluation[]; +}) { const [searchQuery, setSearchQuery] = useState(""); const [selectedDecision, setSelectedDecision] = useState("ALL"); + /** The decision the engine currently produces - not the hard-coded seed label. */ + const engineDecision = (scenario: Scenario) => + evaluations.find((evaluation) => evaluation.scenario.id === scenario.id)?.actual ?? + scenario.expectedDecision; + const filteredScenarios = useMemo(() => { + const evaluationByScenario = new Map( + evaluations.map((evaluation) => [evaluation.scenario.id, evaluation]) + ); + return demoScenarios.filter((scenario) => { const query = searchQuery.trim().toLowerCase(); const matchesSearch = @@ -21,12 +36,14 @@ export function ScenariosCatalog() { scenario.action.domain.toLowerCase().includes(query) || (scenario.action.tool?.toLowerCase().includes(query) ?? false); + const actualDecision = + evaluationByScenario.get(scenario.id)?.actual ?? scenario.expectedDecision; const matchesDecision = - selectedDecision === "ALL" || scenario.expectedDecision === selectedDecision; + selectedDecision === "ALL" || actualDecision === selectedDecision; return matchesSearch && matchesDecision; }); - }, [searchQuery, selectedDecision]); + }, [evaluations, searchQuery, selectedDecision]); const clearFilters = () => { setSearchQuery(""); @@ -96,10 +113,14 @@ export function ScenariosCatalog() { ) : (
{filteredScenarios.map((scenario) => ( -
+

{scenario.title}

- +

{scenario.description}

@@ -115,6 +136,10 @@ export function ScenariosCatalog() {
Amount
{scenario.action.amountXLM} XLM
+
+
Engine decision
+
{engineDecision(scenario)}
+
))} diff --git a/src/lib/auth/use-auth-session.ts b/src/lib/auth/use-auth-session.ts index 88892e59..c30de34d 100644 --- a/src/lib/auth/use-auth-session.ts +++ b/src/lib/auth/use-auth-session.ts @@ -1,6 +1,6 @@ "use client"; -import { useEffect, useState } from "react"; +import { useCallback, useEffect, useState } from "react"; type SessionPayload = { authenticated?: boolean; @@ -65,7 +65,7 @@ export function useAuthSession() { void fetchSession(); }, []); - const refresh = async () => { + const refresh = useCallback(async () => { setLoading(true); try { diff --git a/src/lib/scenarios/evaluate.ts b/src/lib/scenarios/evaluate.ts new file mode 100644 index 00000000..f672ab48 --- /dev/null +++ b/src/lib/scenarios/evaluate.ts @@ -0,0 +1,165 @@ +import { evaluateDecision } from "@/lib/decision/engine"; +import { defaultPolicyConfig } from "@/lib/policy/engine"; +import { defaultDailyUsage, demoScenarios } from "@/lib/scenarios/seed"; +import type { + DecisionResult, + DecisionType, + PolicyConfig, + Scenario, +} from "@/lib/types/domain"; + +/** + * A seed entry as it may arrive at runtime. TypeScript guarantees + * `expectedDecision` on `Scenario`, but seed data can be mutated or generated + * at runtime, so drift validation must tolerate a missing expectation and + * report it instead of crashing. + */ +export type SeedScenarioInput = Omit & { + expectedDecision?: DecisionType | undefined; +}; + +/** One seeded scenario run through the decision engine. */ +export interface ScenarioEvaluation { + scenario: Scenario; + /** Expectation declared by the seed. */ + expected: DecisionType; + /** Decision produced by the engine. */ + actual: DecisionType; + /** Full engine result (triggers, findings, risk score, analyzer health). */ + result: DecisionResult; + /** True when the engine decision matches the seed expectation. */ + matches: boolean; +} + +export type ScenarioPackIssueKind = + | "missing-expected-decision" + | "decision-mismatch"; + +export interface ScenarioPackValidationIssue { + scenarioId: string; + title: string; + kind: ScenarioPackIssueKind; + message: string; +} + +/** + * Policy used to evaluate the catalog: the default policy with the allowed + * hours window removed so results are deterministic regardless of the wall + * clock the suite or the page render runs on. + */ +export const scenarioEvaluationPolicy: PolicyConfig = { + ...defaultPolicyConfig, + allowedHours: undefined, +}; + +/** Usage snapshot used for catalog evaluation (matches the seeded demo state). */ +export const scenarioEvaluationUsage = defaultDailyUsage; + +/** + * Evaluate a single seeded scenario through the decision engine using the + * deterministic catalog policy and usage snapshot. Pure computation: it never + * touches Horizon or any other network dependency. + */ +export async function evaluateScenario(id: string): Promise { + const scenario = demoScenarios.find((entry) => entry.id === id); + if (!scenario) { + throw new Error(`Scenario "${id}" not found in the seed catalog.`); + } + + const result = await evaluateDecision( + scenario.action, + scenarioEvaluationPolicy, + scenarioEvaluationUsage, + ); + + return { + scenario, + expected: scenario.expectedDecision, + actual: result.decision, + result, + matches: result.decision === scenario.expectedDecision, + }; +} + +/** + * Run every seeded scenario through the decision engine. The catalog, the + * seed data, and the engine can change separately - this is the single source + * of truth for what the engine currently decides, consumed by both the drift + * tests and the scenarios page. + */ +export async function evaluateScenarioCatalog(): Promise< + ScenarioEvaluation[] +> { + const evaluations: ScenarioEvaluation[] = []; + for (const scenario of demoScenarios) { + evaluations.push(await evaluateScenario(scenario.id)); + } + return evaluations; +} + +/** Human-readable drift report for one evaluation, including engine rule ids. */ +export function formatScenarioDrift(evaluation: ScenarioEvaluation): string { + const policies = + evaluation.result.triggeredPolicies.map((trigger) => trigger.code).join(", ") || + "none"; + const findings = evaluation.result.riskFindings + .map((finding) => finding.code) + .join(", ") || "none"; + + return ( + `Scenario "${evaluation.scenario.id}" ("${evaluation.scenario.title}") drift: ` + + `seed expects ${evaluation.expected} but engine returned ${evaluation.actual} ` + + `(riskScore ${evaluation.result.riskScore}; policies: ${policies}; findings: ${findings}). ` + + "Update src/lib/scenarios/seed.ts or fix the decision engine." + ); +} + +/** + * Validate the seed catalog against engine output. + * + * Reports an issue when a seed entry has no expected decision at all, and when + * the engine decision no longer matches the seeded expectation. Without + * evaluations only the missing-expectation check can run. + */ +export function validateScenarioPack(params: { + scenarios: ReadonlyArray; + evaluations?: ReadonlyArray; +}): ScenarioPackValidationIssue[] { + const issues: ScenarioPackValidationIssue[] = []; + const evaluationById = new Map( + (params.evaluations ?? []).map( + (evaluation) => [evaluation.scenario.id, evaluation] as const, + ), + ); + + for (const scenario of params.scenarios) { + if (!scenario.expectedDecision) { + issues.push({ + scenarioId: scenario.id, + title: scenario.title, + kind: "missing-expected-decision", + message: + `Scenario "${scenario.id}" ("${scenario.title}") has no expected decision. ` + + "Set expectedDecision in src/lib/scenarios/seed.ts.", + }); + continue; + } + + const evaluation = evaluationById.get(scenario.id); + if (!evaluation || evaluation.actual === scenario.expectedDecision) { + continue; + } + + issues.push({ + scenarioId: scenario.id, + title: scenario.title, + kind: "decision-mismatch", + message: formatScenarioDrift({ + ...evaluation, + expected: scenario.expectedDecision, + }), + }); + } + + return issues; +} diff --git a/src/lib/scenarios/scenario-pack.test.ts b/src/lib/scenarios/scenario-pack.test.ts index ed98e61d..7b3ca109 100644 --- a/src/lib/scenarios/scenario-pack.test.ts +++ b/src/lib/scenarios/scenario-pack.test.ts @@ -1,9 +1,15 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { evaluateDecision } from "@/lib/decision/engine"; import { defaultPolicyConfig } from "@/lib/policy/engine"; +import { + evaluateScenarioCatalog, + validateScenarioPack, + type ScenarioEvaluation, + type SeedScenarioInput, +} from "@/lib/scenarios/evaluate"; import { demoScenarios } from "@/lib/scenarios/seed"; -import type { DailyUsage, PolicyConfig } from "@/lib/types/domain"; +import type { DailyUsage, PolicyConfig, Scenario } from "@/lib/types/domain"; const noTimePolicy: PolicyConfig = { ...defaultPolicyConfig, allowedHours: undefined }; @@ -185,3 +191,81 @@ describe("policy pack regression suite", () => { }); }); }); + +describe("scenario pack drift guard", () => { + const originalBlocklistUrl = process.env.FORTEXA_BLOCKLIST_URL; + + const runWithoutNetwork = async (): Promise => { + const fetchSpy = vi + .spyOn(globalThis, "fetch") + .mockRejectedValue(new Error("network access is forbidden in the scenario pack suite")); + try { + return await evaluateScenarioCatalog(); + } finally { + fetchSpy.mockRestore(); + } + }; + + it("every seed entry declares an expected decision", () => { + const issues = validateScenarioPack({ scenarios: demoScenarios }); + const missing = issues.filter((issue) => issue.kind === "missing-expected-decision"); + expect(missing).toEqual([]); + }); + + it("matches every seeded scenario decision against the engine", async () => { + const evaluations = await runWithoutNetwork(); + expect(evaluations).toHaveLength(demoScenarios.length); + + const issues = validateScenarioPack({ scenarios: demoScenarios, evaluations }); + expect( + issues, + issues.map((issue) => issue.message).join("\n") + ).toEqual([]); + + for (const evaluation of evaluations) { + expect(evaluation.matches, `scenario ${evaluation.scenario.id} drifted`).toBe(true); + } + }); + + it("fails when a seeded expected decision is mutated away from engine output", async () => { + const evaluations = await runWithoutNetwork(); + + const mutated: Scenario[] = demoScenarios.map((scenario, index) => + index === 0 + ? { ...scenario, expectedDecision: "BLOCK" as Scenario["expectedDecision"] } + : scenario + ); + + const issues = validateScenarioPack({ scenarios: mutated, evaluations }); + const mismatch = issues.find((issue) => issue.kind === "decision-mismatch"); + + expect(mismatch).toBeDefined(); + expect(mismatch!.scenarioId).toBe("safe-research-payment"); + expect(mismatch!.message).toContain("seed expects BLOCK"); + expect(mismatch!.message).toContain("engine returned APPROVE"); + expect(mismatch!.message).toContain("src/lib/scenarios/seed.ts"); + }); + + it("fails when a seed entry has no expected decision", () => { + const broken: SeedScenarioInput[] = [ + { ...demoScenarios[0]!, expectedDecision: undefined }, + ]; + + const issues = validateScenarioPack({ scenarios: broken }); + expect(issues).toHaveLength(1); + expect(issues[0]!.kind).toBe("missing-expected-decision"); + expect(issues[0]!.scenarioId).toBe("safe-research-payment"); + }); + + it("evaluates the catalog without contacting Horizon or any network endpoint", async () => { + delete process.env.FORTEXA_BLOCKLIST_URL; + try { + const evaluations = await runWithoutNetwork(); + expect(evaluations).toHaveLength(demoScenarios.length); + } finally { + if (originalBlocklistUrl !== undefined) { + process.env.FORTEXA_BLOCKLIST_URL = originalBlocklistUrl; + } + } + }); +}); diff --git a/src/lib/security/analyzer.ts b/src/lib/security/analyzer.ts index ac2fe028..b4f05f6f 100644 --- a/src/lib/security/analyzer.ts +++ b/src/lib/security/analyzer.ts @@ -157,55 +157,57 @@ function blocklistCheck( return []; } +type BlocklistFetchStatus = { + blocked: boolean; + timedOut: boolean; + error?: string; +}; + /** - * Fetch blocklist with timeout support. Returns findings if successful, empty array if blocked/timed out/failed. - * Returns status indicating what happened. + * Fetch the external blocklist with a timeout guard. Never throws: failures + * are surfaced through the returned status so the analyzer can degrade + * gracefully instead of failing the whole evaluation. When no blocklist URL is + * configured the fetch is skipped entirely and an empty list is returned. */ async function fetchBlocklistWithTimeout(): Promise<{ blocklist: string[]; - status: { blocked: boolean; timedOut: boolean; error?: string }; + status: BlocklistFetchStatus; }> { - try { - const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), timeoutMs); + if (!process.env.FORTEXA_BLOCKLIST_URL) { + return { blocklist: [], status: { blocked: false, timedOut: false } }; + } + + const timeoutMs = getAnalyzerConfig().blocklistTimeoutMs; + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), timeoutMs); - try { - const blocklist = await fetchBlocklist(); - clearTimeout(timeoutId); - const health = getBlocklistHealth(); - if (health.lastError) { - return { - blocklist, - status: { - blocked: true, - timedOut: false, - error: health.lastError, - }, - }; - } - return { blocklist, status: { blocked: false, timedOut: false } }; - } finally { - clearTimeout(timeoutId); + try { + const blocklist = await fetchBlocklist(); + const health = getBlocklistHealth(); + if (health.lastError) { + return { + blocklist, + status: { blocked: true, timedOut: false, error: health.lastError }, + }; } + return { blocklist, status: { blocked: false, timedOut: false } }; } catch (err) { - const isTimeout = err instanceof Error && err.name === "AbortError"; - - // fetchBlocklist swallows errors internally, so check health for failures - const health = getBlocklistHealth(); - if (health.configured && health.lastError) { - const isTimeout = - /abort|timeout/i.test(health.lastError); + const message = err instanceof Error ? err.message : "Unknown fetch error"; + const health = getBlocklistHealth(); + const timedOut = + (err instanceof Error && err.name === "AbortError") || + /abort|timeout/i.test(health.lastError ?? message); return { - blocklist, + blocklist: [], status: { blocked: true, - timedOut: isTimeout, - error: health.lastError, + timedOut, + error: health.lastError ?? message, }, }; + } finally { + clearTimeout(timeoutId); } - - return { blocklist, status: { blocked: false, timedOut: false } }; } export async function evaluateSecurity( @@ -229,7 +231,7 @@ export async function evaluateSecurity( await fetchBlocklistWithTimeout(); if (blocklistFetchStatus.timedOut) { - analyzerStatus.blocklistStatus = "error"; + analyzerStatus.blocklistStatus = "timeout"; analyzerStatus.blocklistTimedOut = true; analyzerStatus.blocklistError = blocklistFetchStatus.error ?? "Blocklist fetch timed out";