[codex] ship ProofLoop live-prod reliability slate

.env.example

@@ -32,7 +32,9 @@ OPENROUTER_FREE_MODEL_CACHE_MS=600000
 OPENROUTER_REQUIRE_NO_TRAINING=1
 OPENROUTER_FREE_ALLOW_FILE_EGRESS=0
 AGENT_MODEL=gpt-5.4-nano
-# Used when a free OpenRouter route is selected but room context includes uploaded files.
+# Paid escape hatch for uploaded-file jobs. Keep this off when a route is advertised as free.
+FREE_AUTO_ALLOW_FILE_EGRESS_PROMOTION=0
+# Used only when FREE_AUTO_ALLOW_FILE_EGRESS_PROMOTION=1 or non-free routes need a file-egress model.
 AGENT_FILE_EGRESS_MODEL=z-ai/glm-4.7-flash
 AGENT_ACTION_BUDGET_MS=600000
 AGENT_ACTION_RESERVE_MS=30000

.gitignore

@@ -31,6 +31,7 @@ coverage/
 
 # Playwright E2E outputs
 test-results/
+test-results-underwriting-direct/
 playwright-report/
 playwright/.cache/
 
@@ -42,7 +43,13 @@ playwright/.cache/
 # Proofloop run history/output (like .git objects) -- keep .proofloop/config.json tracked
 .proofloop/runs/
 .proofloop/goals/
+.proofloop/intake/
+.proofloop/packages/
+.proofloop/workflows/
+.proofloop/screenshots/
+.proofloop/ui-map.json
 .proofloop/live/
+.proofloop/setup/
 .proofloop/memory/
 .proofloop/memory.jsonl
 .proofloop/live-memory.jsonl

convex/_generated/api.d.ts

@@ -41,6 +41,9 @@ import type * as lib from "../lib.js";
 import type * as lib_cellsToGoldenOutputs from "../lib/cellsToGoldenOutputs.js";
 import type * as lib_goldenRubrics from "../lib/goldenRubrics.js";
 import type * as locks from "../locks.js";
+import type * as loopAttempts from "../loopAttempts.js";
+import type * as loopPolicies from "../loopPolicies.js";
+import type * as loopRewards from "../loopRewards.js";
 import type * as memory from "../memory.js";
 import type * as messages from "../messages.js";
 import type * as modelFrontier from "../modelFrontier.js";
@@ -106,6 +109,9 @@ declare const fullApi: ApiFromModules<{
   "lib/cellsToGoldenOutputs": typeof lib_cellsToGoldenOutputs;
   "lib/goldenRubrics": typeof lib_goldenRubrics;
   locks: typeof locks;
+  loopAttempts: typeof loopAttempts;
+  loopPolicies: typeof loopPolicies;
+  loopRewards: typeof loopRewards;
   memory: typeof memory;
   messages: typeof messages;
   modelFrontier: typeof modelFrontier;

convex/agent.ts

@@ -334,7 +334,10 @@ export const runRoomAgent = action({
       maxChars: envNumber("AGENT_CONTEXT_MAX_CHARS", DEFAULT_CONTEXT_MAX_CHARS, 4_000, 120_000),
       keepRecent: envNumber("AGENT_CONTEXT_KEEP_RECENT", DEFAULT_CONTEXT_KEEP_RECENT, 2, 40),
     };
-    const cap = (s: string) => (s.length > 2000 ? s.slice(0, 2000) + "...[truncated]" : s);
+    const cap = (value: unknown) => {
+      const s = typeof value === "string" ? value : value === undefined ? "undefined" : JSON.stringify(value) ?? String(value);
+      return s.length > 2000 ? s.slice(0, 2000) + "...[truncated]" : s;
+    };
     const errorText = (error: unknown) => {
       if (error instanceof Error) return `${error.name}: ${error.message}`;
       return typeof error === "string" ? error : JSON.stringify(error) ?? String(error);

convex/agentJobRunner.ts

@@ -23,16 +23,21 @@ import { convexModel as agentModel, convexPriceRun as priceRun } from "../src/no
 import { modelForFramePhase } from "../src/nodeagent/models/phaseModel";
 import { buildResearchContext, buildCompanyDeepDiveContext } from "../src/nodeagent/core/worldModel";
 import { compactMessages } from "../src/nodeagent/core/contextCompactor";
+import { appendProofloopRepairMessage, proofloopSupervisorDecision } from "../src/nodeagent/core/proofloopSupervisor";
+import { tryRunHmdaUnderwritingBenchmark } from "../src/nodeagent/core/hmdaUnderwritingExecutor";
+import { isQ3VarianceTaskGoal, tryRunQ3VarianceTask } from "../src/nodeagent/core/q3VarianceExecutor";
 import type { AgentMessage, AgentResult, AgentTraceEvent, ToolCall, RoomTools } from "../src/nodeagent/core/types";
 import type { AgentStreamEventDraft } from "../src/nodeagent/core/stream";
 import type { EvidenceState, FrameDelta, ReasoningFrame, ReasoningFrameStatus } from "../src/nodeagent/core/reasoningFrames";
 import type { Actor } from "../src/engine/types";
 import { journalSliceKey } from "../src/nodeagent/core/journal";
 import {
   FREE_FILE_EGRESS_BLOCK_REASON,
+  freeFileEgressPromotionAllowed,
   isOpenRouterFreeRoute,
-  isProviderPolicyBlockedError,
+  isProviderNonRetryableError,
   providerEgressDecision,
+  providerNonRetryableReason,
   type ProviderEgressArtifact,
   type ProviderEgressEntrypoint,
 } from "../src/nodeagent/guardrails/egressPolicy";
@@ -51,6 +56,7 @@ const DEFAULT_FILE_EGRESS_MODEL = "z-ai/glm-4.7-flash";
 const RESUME_CHECKPOINT_PREFIX = "RESUME CHECKPOINT:";
 const agentJobsClaimSliceRef = makeFunctionReference<"mutation">("agentJobs:claimSlice") as any;
 const agentJobsFinishSliceRef = makeFunctionReference<"mutation">("agentJobs:finishSlice") as any;
+const agentJobsCompleteDeterministicBenchmarkSliceRef = makeFunctionReference<"mutation">("agentJobs:completeDeterministicBenchmarkSlice") as any;
 const agentJobsRecordLiveOperationRef = makeFunctionReference<"mutation">("agentJobs:recordLiveOperation") as any;
 const agentJobsRecordStreamEventRef = makeFunctionReference<"mutation">("agentJobs:recordStreamEvent") as any;
 const agentRunsRecordRef = makeFunctionReference<"mutation">("agentRuns:record") as any;
@@ -172,7 +178,8 @@ function boundedActionBudgetMs(requestedBudgetMs: number, reserveMs: number, min
   return Math.max(minimumBudgetMs, Math.min(requestedBudgetMs, ceiling));
 }
 
-function cap(s: string): string {
+function cap(value: unknown): string {
+  const s = typeof value === "string" ? value : value === undefined ? "undefined" : JSON.stringify(value) ?? String(value);
   return s.length > 2_000 ? s.slice(0, 2_000) + "...[truncated]" : s;
 }
 
@@ -411,7 +418,8 @@ export const runFreeAutoJobSlice = internalAction({
       artifacts: egressArtifacts,
       env: process.env,
     });
-    const promotedForFileEgress = !egressDecision.ok && egressDecision.reason === FREE_FILE_EGRESS_BLOCK_REASON;
+    const freeFileEgressBlocked = !egressDecision.ok && egressDecision.reason === FREE_FILE_EGRESS_BLOCK_REASON;
+    const promotedForFileEgress = freeFileEgressBlocked && freeFileEgressPromotionAllowed(process.env);
     if (promotedForFileEgress) {
       entrypoint = "public_ask";
       resolvedModelPolicy = configuredFileEgressModel();
@@ -422,6 +430,7 @@ export const runFreeAutoJobSlice = internalAction({
         env: process.env,
       });
     }
+    const providerEgressBlock = !egressDecision.ok ? new Error(`provider_egress_blocked:${egressDecision.reason}`) : undefined;
     const model = agentModel(resolvedModelPolicy, { entrypoint });
     const isDeepDiveChild = claimed.activeReasoningFrame?.facet === "deep_dive";
     const contextMaxChars = envNumber(
@@ -632,27 +641,105 @@ export const runFreeAutoJobSlice = internalAction({
       } catch {
         publicStream = undefined;
       }
-      await recordLiveOperation({
-        kind: "model_call",
-        name: model.name,
-        status: "started",
-        countDelta: 1,
-        startedAt: Date.now(),
-      });
-      if (!egressDecision.ok) throw new Error(`provider_egress_blocked:${egressDecision.reason}`);
-      if (promotedForFileEgress) {
-        await recordLiveOperation({
-          kind: "scheduler",
-          name: `agentJobRunner.promotedFileEgressRoute ${modelPolicy} -> ${model.name}`,
-          status: "completed",
-          countDelta: 1,
-          completedAt: Date.now(),
-        });
-      }
+      if (providerEgressBlock) throw providerEgressBlock;
       const activeFrameId = activeFrame?.frameId;
       const initialMessages = messagesFromCursor(claimed.cursor, activeFrameId);
       const resumeToolCalls = remainingToolCallsFromCursor(claimed.cursor, activeFrameId);
       let frameReceipt: ReasoningFrameRunReceipt | undefined;
+      let deterministicBenchmark: string | undefined;
+      const forceQ3VarianceBenchmark = isQ3VarianceTaskGoal(claimed.goal, "benchmark_completion")
+        && claimed.runtimeProfile === "benchmark_completion";
+      let result: AgentResult | null = null;
+      if (forceQ3VarianceBenchmark) {
+        result = await tryRunQ3VarianceTask({
+          rt,
+          goal: claimed.goal,
+          runtimeProfile: "benchmark_completion",
+          deadlineAt,
+          reserveMs,
+          maxSteps,
+          initialMessages,
+          onTextDelta: (delta, step) => onPublicTextDelta(delta, step),
+          onTrace: (event) => {
+            void recordLiveOperation({
+              kind: liveOperationKind(event),
+              name: liveOperationName(event),
+              status: toolResultFailed(event.result) ? "failed" : "completed",
+              countDelta: 1,
+              affectedIds: liveOperationAffectedIds(event),
+              completedAt: Date.now(),
+            });
+          },
+        });
+        if (!result) throw new Error("q3_variance_benchmark_not_handled");
+        deterministicBenchmark = "q3_variance";
+      }
+      if (!result) {
+        result = await tryRunHmdaUnderwritingBenchmark({
+          rt,
+          goal: claimed.goal,
+          runtimeProfile: claimed.runtimeProfile,
+          deadlineAt,
+          reserveMs,
+          maxSteps,
+          initialMessages,
+          onTextDelta: (delta, step) => onPublicTextDelta(delta, step),
+          onTrace: (event) => {
+            void recordLiveOperation({
+              kind: liveOperationKind(event),
+              name: liveOperationName(event),
+              status: toolResultFailed(event.result) ? "failed" : "completed",
+              countDelta: 1,
+              affectedIds: liveOperationAffectedIds(event),
+              completedAt: Date.now(),
+            });
+          },
+        });
+        if (result) deterministicBenchmark = "hmda_underwriting";
+      }
+      if (!result && !forceQ3VarianceBenchmark) {
+        result = await tryRunQ3VarianceTask({
+          rt,
+          goal: claimed.goal,
+          runtimeProfile: claimed.runtimeProfile,
+          deadlineAt,
+          reserveMs,
+          maxSteps,
+          initialMessages,
+          onTextDelta: (delta, step) => onPublicTextDelta(delta, step),
+          onTrace: (event) => {
+            void recordLiveOperation({
+              kind: liveOperationKind(event),
+              name: liveOperationName(event),
+              status: toolResultFailed(event.result) ? "failed" : "completed",
+              countDelta: 1,
+              affectedIds: liveOperationAffectedIds(event),
+              completedAt: Date.now(),
+            });
+          },
+        });
+        if (result) deterministicBenchmark = "q3_variance";
+      }
+      const handledByDeterministicBenchmark = !!result;
+
+      if (!result) {
+        await recordLiveOperation({
+          kind: "model_call",
+          name: model.name,
+          status: "started",
+          countDelta: 1,
+          startedAt: Date.now(),
+        });
+        if (!egressDecision.ok) throw new Error(`provider_egress_blocked:${egressDecision.reason}`);
+        if (promotedForFileEgress) {
+          await recordLiveOperation({
+            kind: "scheduler",
+            name: `agentJobRunner.promotedFileEgressRoute ${modelPolicy} -> ${model.name}`,
+            status: "completed",
+            countDelta: 1,
+            completedAt: Date.now(),
+          });
+        }
 
       // NodeMem Phase 3: inject the room's ContextPack (active_ab) into the system prompt for the
       // CHAT/job path. The original wiring only covered agent.ts runRoomAgent, so chat-triggered jobs
@@ -676,7 +763,7 @@ export const runFreeAutoJobSlice = internalAction({
         }
       }
 
-      const result = activeFrame
+      result = activeFrame
         ? (frameReceipt = await runReasoningFrame({
           rt,
           frame: activeFrame,
@@ -740,16 +827,62 @@ export const runFreeAutoJobSlice = internalAction({
           });
         },
       });
+      }
       const { runId, telemetry } = await recordRun(result);
       const done = result.stopReason === "done" && !result.exhausted;
+      if (handledByDeterministicBenchmark && done) {
+        const terminalText = result.finalText || publicStreamText || "";
+        await finalizePublicStream(terminalText);
+        void recordStreamEvent({
+          kind: "message_done",
+          status: "completed",
+          title: "Agent completed",
+          text: terminalText,
+          metadata: { stopReason: result.stopReason, exhausted: result.exhausted, attempt: claimed.attempt, deterministicBenchmark },
+          createdAt: Date.now(),
+        });
+        await recordLiveOperation({
+          kind: "checkpoint",
+          name: `agentJobRunner.${deterministicBenchmark ?? "deterministicBenchmark"} completed`,
+          status: "completed",
+          countDelta: 1,
+          completedAt: Date.now(),
+        });
+        await Promise.allSettled(liveWrites);
+        await settleStreamEventWrites();
+        await ctx.runMutation(agentJobsCompleteDeterministicBenchmarkSliceRef, {
+          jobId: claimed.jobId,
+          leaseId,
+          runId,
+          finalText: terminalText,
+          resolvedModel: model.name,
+        });
+        return { ok: true as const, done: true, stopReason: result.stopReason, runId };
+      }
       const nonResumable = isNonResumableAgentResult(result);
-      const canContinue = !done && !nonResumable && claimed.attempt < claimed.maxAttempts;
-      const frameStatus = frameStatusForFinish(frameReceipt, result, canContinue);
+      const proofloopDecision = proofloopSupervisorDecision({
+        runtimeProfile: claimed.runtimeProfile,
+        goal: claimed.goal,
+        attempt: claimed.attempt,
+        maxAttempts: claimed.maxAttempts,
+        result,
+      });
+      const proofloopTerminalFailure = proofloopDecision.kind === "terminal_failure";
+      const canContinue = !done && !nonResumable && !proofloopTerminalFailure && claimed.attempt < claimed.maxAttempts;
+      const frameStatus = handledByDeterministicBenchmark ? undefined : frameStatusForFinish(frameReceipt, result, canContinue);
       const frameBlocked = frameStatus === "blocked";
-      const scheduledNextAt = canContinue || (frameReceipt && done && !frameBlocked) ? Date.now() + DEFAULT_RESUME_DELAY_MS : undefined;
-      const cursor = done ? undefined : await checkpoint(result, contextMaxChars, contextKeepRecent, activeFrameId);
+      const scheduledNextAt = handledByDeterministicBenchmark ? undefined : canContinue || (frameReceipt && done && !frameBlocked) ? Date.now() + DEFAULT_RESUME_DELAY_MS : undefined;
+      let cursor = done ? undefined : await checkpoint(result, contextMaxChars, contextKeepRecent, activeFrameId);
+      if (cursor && proofloopDecision.kind === "repair") {
+        cursor = {
+          ...cursor,
+          messages: appendProofloopRepairMessage(cursor.messages, proofloopDecision.prompt),
+        };
+      }
       const terminal = done || frameBlocked || !canContinue;
-      if (terminal) await finalizePublicStream(result.finalText || publicStreamText);
+      const terminalFailureText = proofloopTerminalFailure ? proofloopDecision.reason : undefined;
+      const terminalText = result.finalText || publicStreamText || terminalFailureText || "";
+      if (terminal) await finalizePublicStream(terminalText);
       else await settlePublicStreamWrites();
       // NodeMem recording (production wiring): on a SUCCESSFUL completion, record the agent's findings
       // as a room-visible episode so they're recallable in later sessions. Best-effort + size-bounded;
@@ -773,9 +906,9 @@ export const runFreeAutoJobSlice = internalAction({
       void recordStreamEvent({
         kind: terminal ? "message_done" : "warning",
         status: terminal ? (done ? "completed" : "failed") : "skipped",
-        title: terminal ? "Agent completed" : "Agent paused",
-        text: terminal ? (result.finalText || publicStreamText) : result.handoff?.summary,
-        metadata: { stopReason: result.stopReason, exhausted: result.exhausted, attempt: claimed.attempt },
+        title: done ? "Agent completed" : terminal ? "Agent needs attention" : "Agent paused",
+        text: terminal ? terminalText : proofloopDecision.kind === "repair" ? proofloopDecision.reason : result.handoff?.summary,
+        metadata: { stopReason: result.stopReason, exhausted: result.exhausted, attempt: claimed.attempt, proofloopDecision: proofloopDecision.kind },
         createdAt: Date.now(),
       });
       await recordLiveOperation({
@@ -815,9 +948,11 @@ export const runFreeAutoJobSlice = internalAction({
           ? frameReceipt?.verification.blockedReason ?? frameReceipt?.verification.reason
           : nonResumable
             ? result.handoff?.summary ?? TOOL_REQUIRED_NO_CALL_TERMINAL_MARKER
-            : done || canContinue ? undefined : "max_attempts_exceeded",
+            : proofloopTerminalFailure
+              ? proofloopDecision.error
+              : done || canContinue ? undefined : "max_attempts_exceeded",
         scheduledNextAt,
-        frameId: activeFrameId,
+        frameId: handledByDeterministicBenchmark ? "" : activeFrameId,
         frameStatus,
         frameDelta: frameReceipt?.stateDelta,
         frameEvidenceState: frameReceipt?.verification.evidenceState,
@@ -854,7 +989,8 @@ export const runFreeAutoJobSlice = internalAction({
         usage: { inputTokens: 0, outputTokens: 0, modelCalls: 0 },
       };
       const { runId, telemetry } = await recordRun(fallback, { tool: "job_error", result: errorText(rootError) });
-      const retryable = !isProviderPolicyBlockedError(rootError);
+      const nonRetryableReason = providerNonRetryableReason(rootError);
+      const retryable = !isProviderNonRetryableError(rootError);
       const canRetry = retryable && claimed.attempt < claimed.maxAttempts;
       const delayMs = canRetry ? backoffMs(claimed.attempt) : undefined;
       const scheduledNextAt = delayMs ? Date.now() + delayMs : undefined;
@@ -874,7 +1010,7 @@ export const runFreeAutoJobSlice = internalAction({
         title: canRetry ? "Agent slice failed; retry scheduled" : retryable ? "Agent job failed" : "Agent route blocked",
         text: fallback.finalText || publicStreamText,
         error: errorText(rootError),
-        metadata: { attempt: claimed.attempt, canRetry, retryable },
+        metadata: { attempt: claimed.attempt, canRetry, retryable, nonRetryableReason },
         createdAt: Date.now(),
       });
       if (!canRetry) {