feat(scenarios): run seed catalog through the decision engine and fail on drift (#216)

scripts/demo-scenarios.ts

@@ -1,22 +1,58 @@
-import { evaluateDecision } from "../src/lib/decision/engine";
-import { defaultPolicyConfig } from "../src/lib/policy/engine";
-import { demoScenarios, defaultDailyUsage } from "../src/lib/scenarios/seed";
-
-(async () => {
-  console.log("Fortexa Scenario Demo Runner\n");
-
-  for (const scenario of demoScenarios) {
-    const result = await evaluateDecision(scenario.action, defaultPolicyConfig, defaultDailyUsage);
-    console.log(`Scenario: ${scenario.title}`);
-    console.log(`Expected: ${scenario.expectedDecision} | Actual: ${result.decision}`);
-    console.log(`Risk Score: ${result.riskScore}`);
-    console.log(`Explanation: ${result.explanation}`);
-    if (result.triggeredPolicies.length > 0) {
-      console.log(`Policies: ${result.triggeredPolicies.map((item) => item.code).join(", ")}`);
-    }
-    if (result.riskFindings.length > 0) {
-      console.log(`Findings: ${result.riskFindings.map((item) => item.code).join(", ")}`);
-    }
-    console.log("---");
-  }
-})();
+import {
+  evaluateScenarioCatalog,
+  validateScenarioPack,
+} from "../src/lib/scenarios/evaluate";
+import { demoScenarios } from "../src/lib/scenarios/seed";
+
+(async () => {
+  console.log("Fortexa Scenario Demo Runner\n");
+
+  const evaluations = await evaluateScenarioCatalog();
+  let driftCount = 0;
+
+  for (const evaluation of evaluations) {
+    console.log(`Scenario: ${evaluation.scenario.title}`);
+    console.log(
+      `Expected: ${evaluation.expected} | Actual: ${evaluation.actual}`,
+    );
+    console.log(`Risk Score: ${evaluation.result.riskScore}`);
+    console.log(`Explanation: ${evaluation.result.explanation}`);
+    if (evaluation.result.triggeredPolicies.length > 0) {
+      console.log(
+        `Policies: ${evaluation.result.triggeredPolicies
+          .map((item) => item.code)
+          .join(", ")}`,
+      );
+    }
+    if (evaluation.result.riskFindings.length > 0) {
+      console.log(
+        `Findings: ${evaluation.result.riskFindings
+          .map((item) => item.code)
+          .join(", ")}`,
+      );
+    }
+    if (!evaluation.matches) {
+      driftCount += 1;
+      console.log(`DRIFT: ${evaluation.scenario.id} no longer matches its seed.`);
+    }
+    console.log("---");
+  }
+
+  const issues = validateScenarioPack({
+    scenarios: demoScenarios,
+    evaluations,
+  });
+  for (const issue of issues) {
+    console.error(`DRIFT: ${issue.message}`);
+  }
+
+  if (driftCount > 0 || issues.length > 0) {
+    console.error(
+      `\n${Math.max(driftCount, issues.length)} seeded scenario(s) drifted from engine output. Failing.`,
+    );
+    process.exitCode = 1;
+    return;
+  }
+
+  console.log("\nAll seeded scenarios match engine output.");
+})();

src/app/settings/page.tsx

@@ -4,6 +4,7 @@ import { ActivityTimeline } from "@/components/activity-timeline";
 import { OpsDashboard } from "@/components/ops-dashboard";
 import { PolicyEditor } from "@/components/policy-editor";
 import { ScenariosCatalog } from "@/components/scenarios-catalog";
+import { evaluateScenarioCatalog } from "@/lib/scenarios/evaluate";
 import { TabNav, type TabItem } from "@/components/ui/tab-nav";
 import { WalletStatusCard } from "@/components/wallet-status-card";
 import { AUTH_COOKIE_KEY, verifySessionToken } from "@/lib/auth/session";
@@ -36,10 +37,9 @@ export default async function SettingsPage({
   const sessionToken = cookieStore.get(AUTH_COOKIE_KEY)?.value;
   const session = sessionToken ? verifySessionToken(sessionToken) : null;
   const userId = session?.userId;
-  const entries = userId ? await listAuditEntries(userId) : [];
-  const network = resolveStellarNetworkConfig();
-  const horizonUrl = getStellarHorizonUrl();
-  const networkPassphrase = getStellarNetworkPassphrase();
+  const entries = userId ? await listAuditEntries(userId) : [];
+  const scenarioEvaluations =
+    activeTab === "scenarios" ? await evaluateScenarioCatalog() : [];
 
   return (
     <div className="mx-auto max-w-6xl space-y-6">
@@ -81,7 +81,9 @@ export default async function SettingsPage({
           </div>
         </div>
       ) : null}
-      {activeTab === "scenarios" ? <ScenariosCatalog /> : null}
+      {activeTab === "scenarios" ? (
+        <ScenariosCatalog evaluations={scenarioEvaluations} />
+      ) : null}
       {activeTab === "ops" ? <OpsDashboard /> : null}
       {activeTab === "activity" ? <ActivityTimeline entries={entries} /> : null}
     </div>