From faae1e88d7f45f4976e3ef7f5a2644a901e736fe Mon Sep 17 00:00:00 2001 From: homen Date: Tue, 14 Jul 2026 16:28:27 -0700 Subject: [PATCH 01/12] fix(nodeagent): harden workbook routing and proof accounting --- convex/agent.ts | 90 ++- convex/agentJobRunner.ts | 148 +++- convex/agentJobs.ts | 262 ++++++- convex/agentRuns.ts | 22 +- convex/credits.ts | 20 +- convex/schema.ts | 14 + ...chmark-ui-spreadsheetbench-generic.spec.ts | 143 +++- src/app/store.tsx | 82 ++- src/nodeagent/core/proofloopSupervisor.ts | 84 ++- src/nodeagent/core/runtime.ts | 200 +++++- src/nodeagent/core/types.ts | 25 +- src/nodeagent/guardrails/workbookWorkflow.ts | 185 ++++- src/nodeagent/models/convexModel.ts | 648 +++++++++++++++--- src/nodeagent/models/phaseModel.ts | 10 + src/nodeagent/models/qualityFailover.ts | 38 +- .../skills/spreadsheet/cellMutator.ts | 28 +- src/ui/RoomShell.tsx | 8 +- tests/agentJobsRuntime.test.ts | 566 ++++++++++++++- tests/agentJobsSource.test.ts | 41 +- tests/agentRuntime.test.ts | 260 ++++++- tests/convexCredits.test.ts | 45 +- tests/convexModelFreeAutoMode.test.ts | 458 ++++++++++++- tests/gatewayAndJournal.test.ts | 56 ++ tests/idempotencyRuntime.test.ts | 59 +- tests/modelAdapterFreeAutoMode.test.ts | 4 +- tests/phaseModel.test.ts | 10 +- tests/proofloopSupervisor.test.ts | 69 ++ tests/qualityFailover.test.ts | 33 +- tests/roomShellTelemetry.test.ts | 47 ++ tests/workArtifacts.test.ts | 4 +- tests/workbookAgentTools.test.ts | 26 + tests/workbookWorkflowHook.test.ts | 427 +++++++++++- 32 files changed, 3696 insertions(+), 416 deletions(-) create mode 100644 tests/roomShellTelemetry.test.ts diff --git a/convex/agent.ts b/convex/agent.ts index b618ba06..d59122cf 100644 --- a/convex/agent.ts +++ b/convex/agent.ts @@ -29,7 +29,8 @@ import type { Actor } from "../src/engine/types"; type RunResult = { finalText: string; jobId: Id<"agentJobs">; roomId: Id<"rooms">; agentId: string; model: string; goal: string; steps: number; toolCalls: number; conflictsSurvived: number; inputTokens: number; outputTokens: number; - costUsd: number; ms: number; exhausted: boolean; stopReason: string; remainingMs: number | null; deadlineAt: number; + cachedInputTokens: number; cacheCreationInputTokens: number; costUsd: number; costKind: "exact" | "estimated"; + ms: number; exhausted: boolean; stopReason: string; remainingMs: number | null; deadlineAt: number; modelCalls: number; runId: Id<"agentRuns"> | null; handoff: unknown | null; }; import { AgentRunError, runAgent } from "../src/nodeagent/core/runtime"; @@ -133,6 +134,10 @@ function liveOperationKind(event: AgentTraceEvent): LiveOperationKind { return "tool_call"; } +function modelToolCallCount(trace: AgentTraceEvent[]): number { + return trace.filter((event) => event.tool !== "handoff" && event.tool !== "compaction").length; +} + function liveOperationName(event: AgentTraceEvent): string { const result = event.result as { error?: unknown; conflict?: unknown; locked?: unknown; pendingApproval?: unknown } | null; const suffix = toolResultFailed(result) ? " failed" : result?.conflict ? " conflict" : result?.locked ? " blocked" : result?.pendingApproval ? " needs review" : ""; @@ -300,7 +305,7 @@ export const runRoomAgent = action({ }); // Release the credit hold immediately — this run was blocked before any spend (no run yet). // (Other early exits before the run, e.g. egress-blocked, are reclaimed by the sweep cron.) - if (creditsEnforced()) await ctx.runMutation(creditsSettleRef, { roomId: a.roomId, reservationKey: creditReservationKey, actualUsd: 0 }); + if (creditsEnforced()) await ctx.runMutation(creditsSettleRef, { roomId: a.roomId, reservationKey: creditReservationKey, actualUsd: 0, costKind: "exact" }); return { finalText, jobId: jobClaim.jobId, @@ -313,7 +318,10 @@ export const runRoomAgent = action({ conflictsSurvived: 0, inputTokens: 0, outputTokens: 0, + cachedInputTokens: 0, + cacheCreationInputTokens: 0, costUsd: 0, + costKind: "exact", ms, exhausted: false, stopReason: "plan_blocked", @@ -402,14 +410,17 @@ export const runRoomAgent = action({ }; const checkpointCursor = async (r: { messages: unknown[]; - handoff?: { remainingToolCalls?: unknown[] }; + handoff?: { remainingToolCalls?: unknown[]; terminalReason?: string }; stopReason: string; + modelRouteState?: unknown; }) => { const compacted = await compactMessages(r.messages as any, compaction); return { messages: compacted.messages, remainingToolCalls: r.handoff?.remainingToolCalls ?? [], stopReason: r.stopReason, + ...(r.handoff?.terminalReason ? { terminalReason: r.handoff.terminalReason } : {}), + ...(r.modelRouteState ? { modelRouteState: r.modelRouteState } : {}), compacted: compacted.compacted, elided: compacted.elided, updatedAt: Date.now(), @@ -468,7 +479,8 @@ export const runRoomAgent = action({ reused: boolean; row: null | { _id: Id<"agentRuns">; model: string; steps: number; toolCalls: number; conflictsSurvived: number; - inputTokens: number; outputTokens: number; costUsd: number; ms: number; exhausted: boolean; + inputTokens: number; outputTokens: number; cachedInputTokens?: number; cacheCreationInputTokens?: number; + costUsd: number; costKind?: "exact" | "estimated"; ms: number; exhausted: boolean; stopReason?: string; remainingMs?: number; deadlineAt?: number; handoff?: unknown; }; }; @@ -478,7 +490,10 @@ export const runRoomAgent = action({ finalText: row.stopReason ? "Deduplicated: an identical run just completed." : "Deduplicated: an identical run is already in progress.", jobId, roomId: a.roomId, agentId: actor.id, model: row.model, goal: a.goal, steps: row.steps, toolCalls: row.toolCalls, conflictsSurvived: row.conflictsSurvived, - inputTokens: row.inputTokens, outputTokens: row.outputTokens, costUsd: row.costUsd, ms: row.ms, exhausted: row.exhausted, + inputTokens: row.inputTokens, outputTokens: row.outputTokens, + cachedInputTokens: row.cachedInputTokens ?? 0, + cacheCreationInputTokens: row.cacheCreationInputTokens ?? 0, + costUsd: row.costUsd, costKind: row.costKind ?? "estimated", ms: row.ms, exhausted: row.exhausted, stopReason: row.stopReason ?? "in_flight", remainingMs: row.remainingMs ?? null, deadlineAt: row.deadlineAt ?? deadlineAt, modelCalls: 0, runId: row._id, handoff: row.handoff ?? null, }; @@ -515,6 +530,7 @@ export const runRoomAgent = action({ kind: "model_call", name: model.name, status: "started", + countDelta: 0, startedAt: Date.now(), }); @@ -524,20 +540,23 @@ export const runRoomAgent = action({ const ms = Date.now() - t0; const inputTokens = partial?.usage.inputTokens ?? 0; const outputTokens = partial?.usage.outputTokens ?? 0; - const costUsd = priceRun(model.name, inputTokens, outputTokens); + const cachedInputTokens = partial?.usage.cachedInputTokens ?? 0; + const cacheCreationInputTokens = partial?.usage.cacheCreationInputTokens ?? 0; + const costUsd = partial?.usage.costUsd ?? priceRun(model.name, inputTokens, outputTokens); + const costKind = partial?.usage.costKind ?? "estimated"; const conflictsSurvived = partial?.trace.filter((t) => t.tool === "edit_cell" && (t.result as { conflict?: boolean })?.conflict).length ?? 0; const telemetry = { roomId: a.roomId, agentId: actor.id, model: model.name, goal: a.goal, - steps: partial?.steps ?? 0, toolCalls: partial?.trace.length ?? 0, conflictsSurvived, - inputTokens, outputTokens, costUsd, ms, exhausted: partial?.exhausted ?? false, + steps: partial?.steps ?? 0, modelCalls: partial?.usage.modelCalls ?? 0, + toolCalls: partial ? modelToolCallCount(partial.trace) : 0, conflictsSurvived, + inputTokens, outputTokens, cachedInputTokens, cacheCreationInputTokens, costUsd, costKind, ms, exhausted: partial?.exhausted ?? false, stopReason: partial?.stopReason ?? "error", remainingMs: partial?.budget.remainingMs, deadlineAt, handoff: partial?.handoff, }; - await ctx.runMutation(agentRunsFinishRef, { runId, model: model.name, steps: telemetry.steps, toolCalls: telemetry.toolCalls, conflictsSurvived, inputTokens, outputTokens, costUsd, ms, exhausted: telemetry.exhausted, stopReason: telemetry.stopReason, remainingMs: telemetry.remainingMs, deadlineAt, handoff: telemetry.handoff }); - // Settle the credit hold with the ACTUAL (failure-path) cost. No-op unless enforced + enrolled. - if (creditsEnforced()) await ctx.runMutation(creditsSettleRef, { roomId: a.roomId, reservationKey: creditReservationKey, actualUsd: costUsd, runId }); + await ctx.runMutation(agentRunsFinishRef, { runId, model: model.name, steps: telemetry.steps, modelCalls: telemetry.modelCalls, toolCalls: telemetry.toolCalls, conflictsSurvived, inputTokens, outputTokens, cachedInputTokens, cacheCreationInputTokens, costUsd, costKind, ms, exhausted: telemetry.exhausted, stopReason: telemetry.stopReason, remainingMs: telemetry.remainingMs, deadlineAt, handoff: telemetry.handoff }); + if (creditsEnforced()) await ctx.runMutation(creditsSettleRef, { roomId: a.roomId, reservationKey: creditReservationKey, actualUsd: costUsd, costKind, runId }); await recordLiveOperation({ kind: "checkpoint", name: "agent.runRoomAgent failed", @@ -557,8 +576,11 @@ export const runRoomAgent = action({ ms, inputTokens, outputTokens, + cachedInputTokens, + cacheCreationInputTokens, costUsd, - modelCalls: partial?.usage.modelCalls ?? 0, + costKind, + modelCalls: telemetry.modelCalls, toolCalls: telemetry.toolCalls, }); const priorSteps = partial?.trace.map(traceStep) ?? []; @@ -637,35 +659,35 @@ export const runRoomAgent = action({ } const ms = Date.now() - t0; - const costUsd = priceRun(model.name, result.usage.inputTokens, result.usage.outputTokens); + const costUsd = result.usage.costUsd ?? priceRun(model.name, result.usage.inputTokens, result.usage.outputTokens); + const costKind = result.usage.costKind ?? "estimated"; const conflictsSurvived = result.trace.filter((t) => t.tool === "edit_cell" && (t.result as { conflict?: boolean })?.conflict).length; const telemetry = { roomId: a.roomId, agentId: actor.id, model: model.name, goal: a.goal, - steps: result.steps, toolCalls: result.trace.length, conflictsSurvived, - inputTokens: result.usage.inputTokens, outputTokens: result.usage.outputTokens, cachedInputTokens: result.usage.cachedInputTokens ?? 0, costUsd, ms, exhausted: result.exhausted, + steps: result.steps, modelCalls: result.usage.modelCalls, toolCalls: modelToolCallCount(result.trace), conflictsSurvived, + inputTokens: result.usage.inputTokens, outputTokens: result.usage.outputTokens, + cachedInputTokens: result.usage.cachedInputTokens ?? 0, + cacheCreationInputTokens: result.usage.cacheCreationInputTokens ?? 0, + costUsd, costKind, ms, exhausted: result.exhausted, stopReason: result.stopReason, remainingMs: result.budget.remainingMs, deadlineAt, handoff: result.handoff, }; // Patch the claimed run row with final telemetry + the APPEND-ONLY step-level trace (audit + trajectory eval). - await ctx.runMutation(agentRunsFinishRef, { runId, model: model.name, steps: telemetry.steps, toolCalls: telemetry.toolCalls, conflictsSurvived, inputTokens: telemetry.inputTokens, outputTokens: telemetry.outputTokens, costUsd, ms, exhausted: telemetry.exhausted, stopReason: telemetry.stopReason, remainingMs: telemetry.remainingMs, deadlineAt, handoff: telemetry.handoff }); - // Settle the credit hold with the ACTUAL cost. No-op unless enforced + enrolled. - if (creditsEnforced()) await ctx.runMutation(creditsSettleRef, { roomId: a.roomId, reservationKey: creditReservationKey, actualUsd: costUsd, runId }); + await ctx.runMutation(agentRunsFinishRef, { runId, model: model.name, steps: telemetry.steps, modelCalls: telemetry.modelCalls, toolCalls: telemetry.toolCalls, conflictsSurvived, inputTokens: telemetry.inputTokens, outputTokens: telemetry.outputTokens, cachedInputTokens: telemetry.cachedInputTokens, cacheCreationInputTokens: telemetry.cacheCreationInputTokens, costUsd, costKind, ms, exhausted: telemetry.exhausted, stopReason: telemetry.stopReason, remainingMs: telemetry.remainingMs, deadlineAt, handoff: telemetry.handoff }); + if (creditsEnforced()) await ctx.runMutation(creditsSettleRef, { roomId: a.roomId, reservationKey: creditReservationKey, actualUsd: costUsd, costKind, runId }); const done = result.stopReason === "done" && !result.exhausted; - const scheduledNextAt = done ? undefined : Date.now() + 5_000; - const cursor = done ? undefined : await checkpointCursor(result); - await recordLiveOperation({ - kind: "model_call", - name: model.name, - status: "completed", - countDelta: result.usage.modelCalls, - completedAt: Date.now(), - }); + const protocolStall = result.handoff?.terminalReason === "protocol_stall"; + const terminal = done || protocolStall; + const scheduledNextAt = terminal ? undefined : Date.now() + 5_000; + const cursor = terminal ? undefined : await checkpointCursor(result); + // finishInteractive writes the authoritative aggregate model_call event. + // Keeping a second nonzero completion event here would double-count provider requests. await recordLiveOperation({ kind: "checkpoint", - name: done ? "agent.runRoomAgent completed" : "agent.runRoomAgent paused", - status: done ? "completed" : "skipped", + name: done ? "agent.runRoomAgent completed" : protocolStall ? "agent.runRoomAgent failed" : "agent.runRoomAgent paused", + status: done ? "completed" : protocolStall ? "failed" : "skipped", countDelta: 1, completedAt: Date.now(), }); @@ -673,19 +695,23 @@ export const runRoomAgent = action({ await ctx.runMutation(agentJobsFinishInteractiveRef, { jobId, runId, - status: done ? "completed" : "paused", + status: done ? "completed" : protocolStall ? "failed" : "paused", finalText: result.finalText, + ...(protocolStall ? { error: "protocol_stall" } : {}), handoff: result.handoff, cursor, scheduledNextAt, - scheduleWorkflow: !done, + scheduleWorkflow: !terminal, resolvedModel: model.name, stopReason: telemetry.stopReason, ms, inputTokens: telemetry.inputTokens, outputTokens: telemetry.outputTokens, + cachedInputTokens: telemetry.cachedInputTokens, + cacheCreationInputTokens: telemetry.cacheCreationInputTokens, costUsd, - modelCalls: result.usage.modelCalls, + costKind, + modelCalls: telemetry.modelCalls, toolCalls: telemetry.toolCalls, }); await ctx.runMutation(agentStepsRecordRef, { diff --git a/convex/agentJobRunner.ts b/convex/agentJobRunner.ts index 90e85f57..8263b2ab 100644 --- a/convex/agentJobRunner.ts +++ b/convex/agentJobRunner.ts @@ -19,13 +19,13 @@ import { SERVER_PRODUCTION_ROOM_TOOLS as PRODUCTION_ROOM_TOOLS } from "../src/no import { MANAGED_LOCK_SYSTEM_PROMPT } from "../src/nodeagent/models/prompts/systemPrompt"; import { injectMemoryIntoSystemPrompt } from "../src/nodemem/memoryContextBuilder"; import { nodeMemInjectionEnabled, nodeMemRecordingEnabled, nodeMemRoomConfigEnabled } from "./nodemem"; -import { convexModel as agentModel, convexPriceRun as priceRun } from "../src/nodeagent/models/convexModel"; -import { modelForFramePhase } from "../src/nodeagent/models/phaseModel"; +import { configuredConvexModelFallbacks, convexModel as agentModel, convexPriceRun as priceRun } from "../src/nodeagent/models/convexModel"; +import { authorizedModelForFramePhase } from "../src/nodeagent/models/phaseModel"; import { buildResearchContext, buildCompanyDeepDiveContext } from "../src/nodeagent/core/worldModel"; import { compactMessages } from "../src/nodeagent/core/contextCompactor"; import { appendProofloopRepairMessage, proofloopSupervisorDecision } from "../src/nodeagent/core/proofloopSupervisor"; import { tryRunHmdaUnderwritingBenchmark } from "../src/nodeagent/core/hmdaUnderwritingExecutor"; -import type { AgentMessage, AgentResult, AgentTraceEvent, ToolCall, RoomTools } from "../src/nodeagent/core/types"; +import type { AgentMessage, AgentModelRouteState, AgentResult, AgentTraceEvent, ToolCall, RoomTools } from "../src/nodeagent/core/types"; import type { AgentStreamEventDraft } from "../src/nodeagent/core/stream"; import type { EvidenceState, FrameDelta, ReasoningFrame, ReasoningFrameStatus } from "../src/nodeagent/core/reasoningFrames"; import type { Actor } from "../src/engine/types"; @@ -121,6 +121,7 @@ type ClaimedReasoningFrame = { type RunTelemetry = { ms: number; costUsd: number; + costKind: "exact" | "estimated"; }; type RunRecord = { @@ -225,6 +226,10 @@ function liveOperationKind(event: AgentTraceEvent): LiveOperationKind { return "tool_call"; } +function modelToolCallCount(trace: AgentTraceEvent[]): number { + return trace.filter((event) => event.tool !== "handoff" && event.tool !== "compaction").length; +} + function liveOperationName(event: AgentTraceEvent): string { const result = event.result as { error?: unknown; conflict?: unknown; locked?: unknown; pendingApproval?: unknown } | null; const suffix = toolResultFailed(result) ? " failed" : result?.conflict ? " conflict" : result?.locked ? " blocked" : result?.pendingApproval ? " needs review" : ""; @@ -299,14 +304,21 @@ function cursorFrameId(cursor: unknown): string | undefined { return typeof value?.frameId === "string" ? value.frameId : undefined; } +function hasProtocolStallMarker(value: unknown): boolean { + if (!value || typeof value !== "object" || Array.isArray(value)) return false; + return (value as { terminalReason?: unknown }).terminalReason === "protocol_stall"; +} + function messagesFromCursor(cursor: unknown, frameId?: string): AgentMessage[] | undefined { const value = cursor as { messages?: unknown } | undefined; + if (hasProtocolStallMarker(cursor)) return undefined; if (frameId && cursorFrameId(cursor) && cursorFrameId(cursor) !== frameId) return undefined; return Array.isArray(value?.messages) ? value.messages as AgentMessage[] : undefined; } function remainingToolCallsFromCursor(cursor: unknown, frameId?: string): ToolCall[] | undefined { const value = cursor as { remainingToolCalls?: unknown } | undefined; + if (hasProtocolStallMarker(cursor)) return undefined; if (frameId && cursorFrameId(cursor) && cursorFrameId(cursor) !== frameId) return undefined; return Array.isArray(value?.remainingToolCalls) ? value.remainingToolCalls as ToolCall[] : undefined; } @@ -353,12 +365,50 @@ function clean>(value: T): T { return out as T; } +function routeStateFromCursor(cursor: unknown): AgentModelRouteState | undefined { + if (!cursor || typeof cursor !== "object" || Array.isArray(cursor)) return undefined; + const value = (cursor as { modelRouteState?: unknown }).modelRouteState; + if (!value || typeof value !== "object" || Array.isArray(value)) return undefined; + const raw = value as { preferredModelId?: unknown; cooldownUntil?: unknown }; + const preferredModelId = typeof raw.preferredModelId === "string" ? raw.preferredModelId : undefined; + const cooldownUntil: Record = {}; + if (raw.cooldownUntil && typeof raw.cooldownUntil === "object" && !Array.isArray(raw.cooldownUntil)) { + for (const [candidateId, until] of Object.entries(raw.cooldownUntil as Record)) { + if (candidateId && typeof until === "number" && Number.isFinite(until)) cooldownUntil[candidateId] = until; + } + } + if (!preferredModelId && Object.keys(cooldownUntil).length === 0) return undefined; + return { preferredModelId, ...(Object.keys(cooldownUntil).length ? { cooldownUntil } : {}) }; +} + +function discardUnpairedToolCalls(messages: AgentMessage[]): AgentMessage[] { + const completedCallIds = new Set(messages + .filter((message) => message.role === "tool" && message.toolCallId) + .map((message) => message.toolCallId as string)); + const sanitized: AgentMessage[] = []; + for (const message of messages) { + if (message.role !== "assistant" || !message.toolCalls?.length) { + sanitized.push(message); + continue; + } + const retainedCalls = message.toolCalls.filter((call) => completedCallIds.has(call.id)); + if (retainedCalls.length === 0 && !message.content.trim()) continue; + sanitized.push({ + ...message, + toolCalls: retainedCalls.length > 0 ? retainedCalls : undefined, + }); + } + return sanitized; +} + async function checkpoint(result: AgentResult, maxChars: number, keepRecent: number, frameId?: string) { + const nonResumable = isNonResumableAgentResult(result); const compacted = await compactMessages(result.messages, { maxChars, keepRecent }); - const remainingToolCalls = result.handoff?.remainingToolCalls ?? []; - let messages = compacted.messages.filter((message) => + const remainingToolCalls = nonResumable ? [] : result.handoff?.remainingToolCalls ?? []; + const checkpointMessages = nonResumable ? discardUnpairedToolCalls(compacted.messages) : compacted.messages; + let messages = checkpointMessages.filter((message) => !(message.role === "user" && message.content?.startsWith(RESUME_CHECKPOINT_PREFIX))); - if (result.handoff && remainingToolCalls.length === 0) { + if (result.handoff && remainingToolCalls.length === 0 && !nonResumable) { const latestProgress = (result.handoff.latestAssistantText || result.finalText || result.handoff.summary || "") .replace(/\s+/g, " ") .trim() @@ -376,6 +426,8 @@ async function checkpoint(result: AgentResult, maxChars: number, keepRecent: num compacted: compacted.compacted, elided: compacted.elided, updatedAt: Date.now(), + ...(result.modelRouteState ? { modelRouteState: result.modelRouteState } : {}), + ...(nonResumable ? { terminalReason: result.handoff?.terminalReason ?? "protocol_stall" } : {}), }; } @@ -412,7 +464,8 @@ function frameStatusForFinish(receipt: ReasoningFrameRunReceipt | undefined, res function isNonResumableAgentResult(result: AgentResult): boolean { const summary = result.handoff?.summary ?? result.finalText ?? ""; - return result.stopReason === "step_budget" && summary.includes(TOOL_REQUIRED_NO_CALL_TERMINAL_MARKER); + return result.handoff?.terminalReason === "protocol_stall" + || (result.stopReason === "step_budget" && summary.includes(TOOL_REQUIRED_NO_CALL_TERMINAL_MARKER)); } export const runFreeAutoJobSlice = internalAction({ @@ -462,7 +515,20 @@ export const runFreeAutoJobSlice = internalAction({ }); } const providerEgressBlock = !egressDecision.ok ? new Error(`provider_egress_blocked:${egressDecision.reason}`) : undefined; - const model = agentModel(resolvedModelPolicy, { entrypoint }); + const fallbackModelsForRoute = (primaryModel: string): string[] => { + if (claimed.routePolicy === "explicit" || primaryModel === "openrouter/free-auto") return []; + return configuredConvexModelFallbacks(primaryModel).filter((candidate) => providerEgressDecision({ + model: candidate, + entrypoint, + artifacts: egressArtifacts, + env: process.env, + }).ok); + }; + const model = agentModel(resolvedModelPolicy, { + entrypoint, + fallbackModelIds: fallbackModelsForRoute(resolvedModelPolicy), + routeState: routeStateFromCursor(claimed.cursor), + }); const isDeepDiveChild = claimed.activeReasoningFrame?.facet === "deep_dive"; const contextMaxChars = envNumber( "FREE_AUTO_JOB_CONTEXT_MAX_CHARS", @@ -484,10 +550,19 @@ export const runFreeAutoJobSlice = internalAction({ // AGENT_ORCHESTRATOR_MODEL; worker phases (execute) use AGENT_WORKER_MODEL. // Falls back to resolvedModelPolicy if env vars not set. const phaseModel = activeFrame - ? modelForFramePhase(activeFrame.phase, resolvedModelPolicy) + ? authorizedModelForFramePhase(activeFrame.phase, resolvedModelPolicy, (candidate) => providerEgressDecision({ + model: candidate, + entrypoint, + artifacts: egressArtifacts, + env: process.env, + }).ok) : resolvedModelPolicy; const phaseAwareModel = phaseModel !== resolvedModelPolicy - ? agentModel(phaseModel, { entrypoint }) + ? agentModel(phaseModel, { + entrypoint, + fallbackModelIds: fallbackModelsForRoute(phaseModel), + routeState: routeStateFromCursor(claimed.cursor), + }) : model; let liveSequence = 1_000 + Math.max(0, claimed.attempt - 1) * 10_000; let streamSequence = 1_000 + Math.max(0, claimed.attempt - 1) * 10_000; @@ -538,7 +613,7 @@ export const runFreeAutoJobSlice = internalAction({ handoff: claimed.handoff ?? null, maxSteps, }), - modelName: () => model.name, + modelName: () => phaseAwareModel.name, }); let publicStream: PublicAgentJobStream | undefined; let publicStreamText = ""; @@ -605,21 +680,25 @@ export const runFreeAutoJobSlice = internalAction({ const recordRun = async (result: AgentResult, extraStep?: { tool: string; result: string }): Promise => { const ms = Date.now() - t0; - const costUsd = priceRun(model.name, result.usage.inputTokens, result.usage.outputTokens); + const costUsd = result.usage.costUsd ?? priceRun(phaseAwareModel.name, result.usage.inputTokens, result.usage.outputTokens); + const costKind = result.usage.costKind ?? "estimated"; const conflictsSurvived = result.trace.filter((t) => t.tool === "edit_cell" && (t.result as { conflict?: boolean })?.conflict).length; const telemetry = { jobId: claimed.jobId, roomId: claimed.roomId, agentId: actor.id, - model: model.name, + model: phaseAwareModel.name, goal: claimed.goal, steps: result.steps, - toolCalls: result.trace.length, + modelCalls: result.usage.modelCalls, + toolCalls: modelToolCallCount(result.trace), conflictsSurvived, inputTokens: result.usage.inputTokens, outputTokens: result.usage.outputTokens, cachedInputTokens: result.usage.cachedInputTokens ?? 0, + cacheCreationInputTokens: result.usage.cacheCreationInputTokens ?? 0, costUsd, + costKind, ms, exhausted: result.exhausted, stopReason: result.stopReason, @@ -649,7 +728,7 @@ export const runFreeAutoJobSlice = internalAction({ agentId: actor.id, steps, }); - return { runId, telemetry }; + return { runId, telemetry: { ...telemetry, costKind } }; }; try { @@ -674,8 +753,9 @@ export const runFreeAutoJobSlice = internalAction({ } if (providerEgressBlock) throw providerEgressBlock; const activeFrameId = activeFrame?.frameId; - const initialMessages = messagesFromCursor(claimed.cursor, activeFrameId); - const resumeToolCalls = remainingToolCallsFromCursor(claimed.cursor, activeFrameId); + const manualRetryAfterProtocolStall = hasProtocolStallMarker(claimed.handoff) || hasProtocolStallMarker(claimed.cursor); + const initialMessages = manualRetryAfterProtocolStall ? undefined : messagesFromCursor(claimed.cursor, activeFrameId); + const resumeToolCalls = manualRetryAfterProtocolStall ? undefined : remainingToolCallsFromCursor(claimed.cursor, activeFrameId); const workbookWorkflowHooks = [createVerifiedWorkbookWorkflowHook()]; let frameReceipt: ReasoningFrameRunReceipt | undefined; let result: AgentResult | null = await tryRunHmdaUnderwritingBenchmark({ @@ -703,9 +783,11 @@ export const runFreeAutoJobSlice = internalAction({ if (!result) { await recordLiveOperation({ kind: "model_call", - name: model.name, + name: phaseAwareModel.name, status: "started", - countDelta: 1, + // A start marker is lifecycle telemetry, not a provider request. The + // completed/failed marker below carries the authoritative request count. + countDelta: 0, startedAt: Date.now(), }); if (!egressDecision.ok) throw new Error(`provider_egress_blocked:${egressDecision.reason}`); @@ -835,7 +917,7 @@ export const runFreeAutoJobSlice = internalAction({ leaseId, runId, finalText: terminalText, - resolvedModel: model.name, + resolvedModel: phaseAwareModel.name, }); return { ok: true as const, done: true, stopReason: result.stopReason, runId }; } @@ -852,7 +934,7 @@ export const runFreeAutoJobSlice = internalAction({ const frameStatus = handledByHmdaBenchmark ? undefined : frameStatusForFinish(frameReceipt, result, canContinue); const frameBlocked = frameStatus === "blocked"; const scheduledNextAt = handledByHmdaBenchmark ? undefined : canContinue || (frameReceipt && done && !frameBlocked) ? Date.now() + DEFAULT_RESUME_DELAY_MS : undefined; - let cursor = done ? undefined : await checkpoint(result, contextMaxChars, contextKeepRecent, activeFrameId); + let cursor = done || nonResumable ? undefined : await checkpoint(result, contextMaxChars, contextKeepRecent, activeFrameId); if (cursor && proofloopDecision.kind === "repair") { cursor = { ...cursor, @@ -893,7 +975,7 @@ export const runFreeAutoJobSlice = internalAction({ }); await recordLiveOperation({ kind: "model_call", - name: model.name, + name: phaseAwareModel.name, status: "completed", countDelta: result.usage.modelCalls, completedAt: Date.now(), @@ -913,13 +995,17 @@ export const runFreeAutoJobSlice = internalAction({ leaseId, attempt: claimed.attempt, status: frameBlocked ? "blocked" : done ? "completed" : canContinue ? "handoff" : "failed", - resolvedModel: model.name, + resolvedModel: phaseAwareModel.name, stopReason: result.stopReason, ms: telemetry.ms, inputTokens: result.usage.inputTokens, outputTokens: result.usage.outputTokens, cachedInputTokens: result.usage.cachedInputTokens ?? 0, + cacheCreationInputTokens: result.usage.cacheCreationInputTokens ?? 0, costUsd: telemetry.costUsd, + costKind: telemetry.costKind, + modelCalls: result.usage.modelCalls, + toolCalls: modelToolCallCount(result.trace), runId, handoff: result.handoff, cursor, @@ -927,7 +1013,7 @@ export const runFreeAutoJobSlice = internalAction({ error: frameBlocked ? frameReceipt?.verification.blockedReason ?? frameReceipt?.verification.reason : nonResumable - ? result.handoff?.summary ?? TOOL_REQUIRED_NO_CALL_TERMINAL_MARKER + ? result.handoff?.terminalReason ?? result.handoff?.summary ?? TOOL_REQUIRED_NO_CALL_TERMINAL_MARKER : proofloopTerminalFailure ? proofloopDecision.error : done || canContinue ? undefined : "max_attempts_exceeded", @@ -1002,6 +1088,13 @@ export const runFreeAutoJobSlice = internalAction({ createdAt: Date.now(), }); } + await recordLiveOperation({ + kind: "model_call", + name: phaseAwareModel.name, + status: "failed", + countDelta: fallback.usage.modelCalls, + completedAt: Date.now(), + }); await recordLiveOperation({ kind: "checkpoint", name: "agentJobRunner.runFreeAutoJobSlice failed", @@ -1015,12 +1108,17 @@ export const runFreeAutoJobSlice = internalAction({ leaseId, attempt: claimed.attempt, status: canRetry ? "retrying" : "failed", - resolvedModel: model.name, + resolvedModel: phaseAwareModel.name, stopReason: fallback.stopReason, ms: telemetry.ms, inputTokens: fallback.usage.inputTokens, outputTokens: fallback.usage.outputTokens, + cachedInputTokens: fallback.usage.cachedInputTokens ?? 0, + cacheCreationInputTokens: fallback.usage.cacheCreationInputTokens ?? 0, costUsd: telemetry.costUsd, + costKind: telemetry.costKind, + modelCalls: fallback.usage.modelCalls, + toolCalls: modelToolCallCount(fallback.trace), runId, error: errorText(rootError), handoff: fallback.handoff, diff --git a/convex/agentJobs.ts b/convex/agentJobs.ts index 691b2cd6..8b9fb49d 100644 --- a/convex/agentJobs.ts +++ b/convex/agentJobs.ts @@ -25,6 +25,7 @@ function companyKeyOf(name: string): string { } const attemptStatusV = v.union(v.literal("completed"), v.literal("handoff"), v.literal("retrying"), v.literal("blocked"), v.literal("failed")); +const costKindV = v.union(v.literal("exact"), v.literal("estimated")); const terminalStatuses = new Set(["completed", "failed", "blocked", "cancelled"]); const entrypointV = v.union( v.literal("public_ask"), @@ -94,6 +95,7 @@ const deltaStatusV = v.union(v.literal("none"), v.literal("minor"), v.literal("m type EntityType = "company" | "person" | "product" | "source" | "metric" | "unknown"; type CacheVisibility = "public" | "private" | "redacted"; type RoomWorkMode = "manual_capture" | "agent_fill" | "bulk_diligence" | "banker_workflow" | "spreadsheet_fill"; +type CostKind = "exact" | "estimated"; type NormalizedRoomWorkEntity = { entityType: EntityType; entityKey: string; displayName: string; website?: string }; type EntityFacetCacheHit = { cacheId: Id<"entityResearchCache">; @@ -136,6 +138,88 @@ type RoomWorkFacetPlan = { status: "queued" | "cached" | "refreshing" | "completed"; }; +function persistedCallCount(value: number | undefined, legacyFallback: number, label: string): number { + const count = value ?? legacyFallback; + if (!Number.isSafeInteger(count) || count < 0) throw new Error(`${label}_must_be_a_non_negative_integer`); + return count; +} + +function aggregateCostKind(previous: unknown, current: CostKind): CostKind { + return previous === "estimated" || current === "estimated" ? "estimated" : "exact"; +} + +type JobAccountingAggregate = { + modelCallCount?: number; + toolCallCount?: number; + inputTokens?: number; + outputTokens?: number; + cachedInputTokens?: number; + cacheCreationInputTokens?: number; + costUsd?: number; + costKind?: CostKind; +}; + +type JobAccountingBaseline = Required; +const MAX_ACCOUNTING_MIGRATION_ATTEMPTS = 1_000; + +async function loadJobAccountingBaseline( + ctx: any, + jobId: Id<"agentJobs">, + job: JobAccountingAggregate, +): Promise { + const aggregateInitialized = job.modelCallCount !== undefined + && job.toolCallCount !== undefined + && job.inputTokens !== undefined + && job.outputTokens !== undefined + && job.cachedInputTokens !== undefined + && job.cacheCreationInputTokens !== undefined + && job.costUsd !== undefined + && job.costKind !== undefined; + if (aggregateInitialized) return job as JobAccountingBaseline; + + // Active jobs created before aggregate accounting can already have durable attempts. + // Rebuild only absent parent fields; a persisted zero is initialized and authoritative. + const attempts = await ctx.db + .query("agentJobAttempts") + .withIndex("by_job", (q: any) => q.eq("jobId", jobId)) + .order("asc") + .take(MAX_ACCOUNTING_MIGRATION_ATTEMPTS + 1); + if (attempts.length > MAX_ACCOUNTING_MIGRATION_ATTEMPTS) { + throw new Error("job_accounting_migration_attempt_limit_exceeded"); + } + const reconstructed = attempts.reduce((sum: JobAccountingBaseline, attempt: any) => { + sum.modelCallCount += persistedCallCount(attempt.modelCalls, 1, "attempt.modelCalls"); + sum.toolCallCount += persistedCallCount(attempt.toolCalls, attempt.inputTokens || attempt.outputTokens ? 1 : 0, "attempt.toolCalls"); + sum.inputTokens += attempt.inputTokens; + sum.outputTokens += attempt.outputTokens; + sum.cachedInputTokens += attempt.cachedInputTokens ?? 0; + sum.cacheCreationInputTokens += attempt.cacheCreationInputTokens ?? 0; + sum.costUsd += attempt.costUsd; + sum.costKind = aggregateCostKind(sum.costKind, attempt.costKind ?? "estimated"); + return sum; + }, { + modelCallCount: 0, + toolCallCount: 0, + inputTokens: 0, + outputTokens: 0, + cachedInputTokens: 0, + cacheCreationInputTokens: 0, + costUsd: 0, + costKind: "exact", + }); + + return { + modelCallCount: job.modelCallCount ?? reconstructed.modelCallCount, + toolCallCount: job.toolCallCount ?? reconstructed.toolCallCount, + inputTokens: job.inputTokens ?? reconstructed.inputTokens, + outputTokens: job.outputTokens ?? reconstructed.outputTokens, + cachedInputTokens: job.cachedInputTokens ?? reconstructed.cachedInputTokens, + cacheCreationInputTokens: job.cacheCreationInputTokens ?? reconstructed.cacheCreationInputTokens, + costUsd: job.costUsd ?? reconstructed.costUsd, + costKind: job.costKind ?? reconstructed.costKind, + }; +} + const MAX_ROOM_WORK_TEXT_CHARS = 20_000; const MAX_ROOM_WORK_ENTITIES = 50; const MAX_ROOM_WORK_FACETS = 16; @@ -945,7 +1029,10 @@ export const finishInteractive = internalMutation({ ms: v.number(), inputTokens: v.number(), outputTokens: v.number(), + cachedInputTokens: v.optional(v.number()), + cacheCreationInputTokens: v.optional(v.number()), costUsd: v.number(), + costKind: v.optional(costKindV), modelCalls: v.number(), toolCalls: v.number(), queryCount: v.optional(v.number()), @@ -956,6 +1043,10 @@ export const finishInteractive = internalMutation({ const job = await ctx.db.get(a.jobId); if (!job) return { ok: false as const, reason: "job_not_found" as const }; if (terminalStatuses.has(job.status) && job.latestRunId) return { ok: true as const, terminal: true as const }; + const modelCalls = persistedCallCount(a.modelCalls, 0, "modelCalls"); + const toolCalls = persistedCallCount(a.toolCalls, 0, "toolCalls"); + const costKind: CostKind = a.costKind ?? "estimated"; + const accounting = await loadJobAccountingBaseline(ctx, a.jobId, job); const now = Date.now(); const attempt = job.attempts + 1; await ctx.db.insert("agentJobAttempts", clean({ @@ -968,16 +1059,21 @@ export const finishInteractive = internalMutation({ ms: a.ms, inputTokens: a.inputTokens, outputTokens: a.outputTokens, + cachedInputTokens: a.cachedInputTokens, + cacheCreationInputTokens: a.cacheCreationInputTokens, costUsd: a.costUsd, + costKind, + modelCalls, + toolCalls, error: a.error, startedAt: now - a.ms, endedAt: now, })); - const baseSequence = (job.actionSliceCount ?? 0) + (job.queryCount ?? 0) + (job.mutationCount ?? 0) + (job.modelCallCount ?? 0) + (job.toolCallCount ?? 0) + (job.schedulerHandoffCount ?? 0) + 2; + const baseSequence = (job.actionSliceCount ?? 0) + (job.queryCount ?? 0) + (job.mutationCount ?? 0) + accounting.modelCallCount + accounting.toolCallCount + (job.schedulerHandoffCount ?? 0) + 2; const eventStatus = a.status === "failed" || a.status === "blocked" ? "failed" : "completed"; await recordOperationEvent(ctx, { jobId: a.jobId, runId: a.runId, sequence: baseSequence, kind: "action", name: "agent.runRoomAgent", countDelta: 1, status: eventStatus, startedAt: now - a.ms, completedAt: now }); - await recordOperationEvent(ctx, { jobId: a.jobId, runId: a.runId, sequence: baseSequence + 1, kind: "model_call", name: a.resolvedModel, countDelta: a.modelCalls, status: eventStatus, startedAt: now - a.ms, completedAt: now }); - await recordOperationEvent(ctx, { jobId: a.jobId, runId: a.runId, sequence: baseSequence + 2, kind: "tool_call", name: "NodeAgent tools", countDelta: a.toolCalls, status: eventStatus, startedAt: now - a.ms, completedAt: now }); + await recordOperationEvent(ctx, { jobId: a.jobId, runId: a.runId, sequence: baseSequence + 1, kind: "model_call", name: a.resolvedModel, countDelta: modelCalls, status: eventStatus, startedAt: now - a.ms, completedAt: now }); + await recordOperationEvent(ctx, { jobId: a.jobId, runId: a.runId, sequence: baseSequence + 2, kind: "tool_call", name: "NodeAgent tools", countDelta: toolCalls, status: eventStatus, startedAt: now - a.ms, completedAt: now }); await recordOperationEvent(ctx, { jobId: a.jobId, runId: a.runId, sequence: baseSequence + 3, kind: "checkpoint", name: "agentJobs.finishInteractive", countDelta: 1, status: "completed", startedAt: now, completedAt: now }); await recordStreamEventRow(ctx, { jobId: a.jobId, @@ -1006,21 +1102,27 @@ export const finishInteractive = internalMutation({ finalText: a.finalText, error: a.error, handoff: a.handoff, - cursor: a.cursor, - nextRunAt: a.scheduledNextAt, + cursor: a.status === "paused" ? a.cursor : null, + nextRunAt: a.status === "paused" ? a.scheduledNextAt : 0, runtime: workflowId ? "workflow" : job.runtime, workflowId, actionSliceCount: (job.actionSliceCount ?? 0) + 1, queryCount: (job.queryCount ?? 0) + (a.queryCount ?? 1), mutationCount: (job.mutationCount ?? 0) + (a.mutationCount ?? 1), - modelCallCount: (job.modelCallCount ?? 0) + a.modelCalls, - toolCallCount: (job.toolCallCount ?? 0) + a.toolCalls, + modelCallCount: accounting.modelCallCount + modelCalls, + toolCallCount: accounting.toolCallCount + toolCalls, + inputTokens: accounting.inputTokens + a.inputTokens, + outputTokens: accounting.outputTokens + a.outputTokens, + cachedInputTokens: accounting.cachedInputTokens + (a.cachedInputTokens ?? 0), + cacheCreationInputTokens: accounting.cacheCreationInputTokens + (a.cacheCreationInputTokens ?? 0), + costUsd: accounting.costUsd + a.costUsd, + costKind: aggregateCostKind(accounting.costKind, costKind), receiptCount: (job.receiptCount ?? 0) + (a.receiptCount ?? 0), schedulerHandoffCount: (job.schedulerHandoffCount ?? 0) + (workflowId ? 1 : 0), leaseId: "", leaseUntil: 0, updatedAt: now, - completedAt: a.status === "completed" ? now : undefined, + completedAt: a.status === "paused" ? undefined : now, }) as any); return { ok: true as const }; }, @@ -1539,6 +1641,7 @@ type DurableStartEntrypoint = "public_ask" | "private_agent" | "free" | "system" type RoutePolicy = "fast_default" | "free_auto" | "top_paid" | "explicit"; type RuntimePolicy = "workflow_sliced"; type AgentRuntimeProfile = "benchmark_completion"; +const PUBLIC_BENCHMARK_COMPLETION_MAX_ATTEMPTS = 12; function inferredRuntimeProfileForGoal(goal: string): AgentRuntimeProfile | undefined { return /\b(benchmark|eval|scorecard|held[- ]out|spreadsheetbench|bankertoolbench|btb|official\s+benchmark)\b/i.test(goal) @@ -1546,12 +1649,59 @@ function inferredRuntimeProfileForGoal(goal: string): AgentRuntimeProfile | unde : undefined; } -function maxAttemptsCeilingForRuntimeProfile(runtimeProfile?: AgentRuntimeProfile): number { - return runtimeProfile === "benchmark_completion" ? 1000 : 100; -} +type DurableJobAttemptIdentity = { + scope?: string; + runtimeProfile?: string; + entrypoint?: string; + modelPolicy?: string; + runtime?: string; + goal?: string; + commandText?: string; + request?: unknown; +}; -function boundedMaxAttempts(requested: number | undefined, fallback: number, runtimeProfile?: AgentRuntimeProfile): number { - return Math.max(1, Math.min(requested ?? fallback, maxAttemptsCeilingForRuntimeProfile(runtimeProfile))); +function isPublicBenchmarkCompletionJob(job: DurableJobAttemptIdentity): boolean { + const request = job.request && typeof job.request === "object" && !Array.isArray(job.request) + ? job.request as Record + : {}; + const scope = job.scope ?? (typeof request.scope === "string" ? request.scope : undefined); + const entrypoint = job.entrypoint ?? (typeof request.entrypoint === "string" ? request.entrypoint : undefined); + // The first public startFreeAuto rows predate scope/entrypoint/request metadata. Keep this + // migration fallback narrow so an unrelated untagged private/system job is never reclassified. + const isLegacyPublicFreeAuto = scope === undefined + && entrypoint === undefined + && job.modelPolicy === "openrouter/free-auto" + && job.runtime === "workflow"; + const isPublic = scope !== undefined + ? scope === "public_room" + : entrypoint === "public_ask" || entrypoint === "free" || entrypoint === "room_work" || isLegacyPublicFreeAuto; + const requestRuntimeProfile = typeof request.runtimeProfile === "string" ? request.runtimeProfile : undefined; + const goal = job.goal + ?? job.commandText + ?? (typeof request.commandText === "string" ? request.commandText : ""); + const isBenchmark = job.runtimeProfile === "benchmark_completion" + || requestRuntimeProfile === "benchmark_completion" + || inferredRuntimeProfileForGoal(goal) === "benchmark_completion"; + return isPublic && isBenchmark; +} + +function maxAttemptsCeilingForJob(job: DurableJobAttemptIdentity & { execution: "inline" | "workflow" }): number { + if (isPublicBenchmarkCompletionJob(job)) return PUBLIC_BENCHMARK_COMPLETION_MAX_ATTEMPTS; + return job.execution === "inline" ? 20 : 100; +} + +function boundedMaxAttempts( + requested: number | undefined, + fallback: number, + job: DurableJobAttemptIdentity & { execution: "inline" | "workflow" }, +): number { + return Math.max(1, Math.min(requested ?? fallback, maxAttemptsCeilingForJob(job))); +} + +function durableMaxAttemptsForJob(job: DurableJobAttemptIdentity, requestedMaxAttempts: number): number { + return isPublicBenchmarkCompletionJob(job) + ? Math.min(requestedMaxAttempts, PUBLIC_BENCHMARK_COMPLETION_MAX_ATTEMPTS) + : requestedMaxAttempts; } const ELEMENT_SCOPED_FRAME_TOOL_ALLOWLIST = ["snapshot", "list_artifacts", "read_range", "search_sheet_context", "write_locked_cell"]; @@ -1873,8 +2023,12 @@ async function startDurableAgentJob(ctx: any, a: DurableStartAgentJobArgs): Prom routePolicy = "explicit"; modelPolicy = configuredFileEgressModel(); } - const defaultMaxAttempts = runtimeProfile === "benchmark_completion" ? 1000 : entrypoint === "free" ? 20 : 20; - const maxAttempts = boundedMaxAttempts(a.maxAttempts, defaultMaxAttempts, runtimeProfile); + const defaultMaxAttempts = execution === "inline" + ? 1 + : runtimeProfile === "benchmark_completion" + ? PUBLIC_BENCHMARK_COMPLETION_MAX_ATTEMPTS + : 20; + const maxAttempts = boundedMaxAttempts(a.maxAttempts, defaultMaxAttempts, { execution, scope, runtimeProfile }); const idempotencyKey = a.idempotencyKey ?? defaultJobIdempotencyKey({ roomId: a.roomId, artifactId: a.artifactId, @@ -1886,6 +2040,28 @@ async function startDurableAgentJob(ctx: any, a: DurableStartAgentJobArgs): Prom const prior = await ctx.db.query("agentJobs").withIndex("by_idempotency", (q: any) => q.eq("idempotencyKey", idempotencyKey)).order("desc").take(5); const reusable = prior.find((job: any) => String(job.roomId) === String(a.roomId) && String(job.artifactId) === String(a.artifactId) && !terminalStatuses.has(job.status)); if (reusable) { + const reusableIdentity = { + ...reusable, + scope: reusable.scope ?? scope, + runtimeProfile: reusable.runtimeProfile ?? runtimeProfile, + entrypoint: reusable.entrypoint ?? entrypoint, + goal: reusable.goal ?? a.goal, + }; + const normalizedMaxAttempts = durableMaxAttemptsForJob(reusableIdentity, reusable.maxAttempts); + if ( + reusable.maxAttempts !== normalizedMaxAttempts + || reusable.scope === undefined + || reusable.runtimeProfile === undefined + || reusable.entrypoint === undefined + ) { + await ctx.db.patch(reusable._id, clean({ + maxAttempts: normalizedMaxAttempts, + scope: reusable.scope ?? scope, + runtimeProfile: reusable.runtimeProfile ?? runtimeProfile, + entrypoint: reusable.entrypoint ?? entrypoint, + updatedAt: now, + })); + } return { jobId: reusable._id as Id<"agentJobs">, reused: true as const, status: reusable.status as string, workflowId: reusable.workflowId as string | undefined, latestRunId: reusable.latestRunId as Id<"agentRuns"> | undefined, modelPolicy: reusable.modelPolicy as string, routePolicy, runtimePolicy }; } @@ -1969,7 +2145,7 @@ async function startDurableAgentJob(ctx: any, a: DurableStartAgentJobArgs): Prom modelPolicy, runtime, attempts: 0, - maxAttempts: execution === "inline" ? Math.max(1, Math.min(a.maxAttempts ?? 1, 20)) : maxAttempts, + maxAttempts, actionSliceCount: 0, queryCount: 0, mutationCount: 1, @@ -2479,7 +2655,12 @@ export const retry = mutation({ } const now = Date.now(); const extra = Math.max(1, Math.min(additionalAttempts ?? 10, 50)); - const maxAttempts = Math.max(job.maxAttempts, job.attempts + extra); + const requestedMaxAttempts = Math.max(job.maxAttempts, job.attempts + extra); + const maxAttempts = durableMaxAttemptsForJob(job, requestedMaxAttempts); + if (job.attempts >= maxAttempts) { + if (job.maxAttempts !== maxAttempts) await ctx.db.patch(jobId, { maxAttempts, updatedAt: now }); + return { ok: false as const, reason: "attempt_limit_reached" as const, maxAttempts }; + } const workflowId = await startWorkflow(ctx, internal.agentWorkflows.freeAutoWorkflow, { jobId }, { onComplete: internal.agentWorkflows.freeAutoWorkflowComplete, context: { jobId }, @@ -2728,6 +2909,19 @@ export const claimSlice = internalMutation({ if (job.status === "completed" || job.status === "failed" || job.status === "blocked" || job.status === "cancelled") return null; const now = Date.now(); if (job.status === "running" && job.leaseUntil && job.leaseUntil > now) return null; + const maxAttempts = durableMaxAttemptsForJob(job, job.maxAttempts); + if (isPublicBenchmarkCompletionJob(job) && job.attempts >= maxAttempts) { + await ctx.db.patch(jobId, { + status: "failed", + maxAttempts, + leaseId: "", + leaseUntil: 0, + error: "max_attempts_exhausted", + completedAt: now, + updatedAt: now, + }); + return null; + } const art = await ctx.db.get(job.artifactId); if (!art || String(art.roomId) !== String(job.roomId)) { @@ -2759,6 +2953,7 @@ export const claimSlice = internalMutation({ await ctx.db.patch(jobId, { status: "running", attempts: attempt, + maxAttempts, leaseId, leaseUntil, activeFrameId: frameClaim.frame?.frameId ?? "", @@ -2813,7 +3008,7 @@ export const claimSlice = internalMutation({ cursor: job.cursor, handoff: job.handoff, attempt, - maxAttempts: job.maxAttempts, + maxAttempts, sessionId: session._id, agentId: session.agentId, agentName: session.agentName, @@ -2975,7 +3170,11 @@ export const finishSlice = internalMutation({ inputTokens: v.number(), outputTokens: v.number(), cachedInputTokens: v.optional(v.number()), + cacheCreationInputTokens: v.optional(v.number()), costUsd: v.number(), + costKind: v.optional(costKindV), + modelCalls: v.optional(v.number()), + toolCalls: v.optional(v.number()), runId: v.optional(v.id("agentRuns")), error: v.optional(v.string()), handoff: v.optional(v.any()), @@ -2992,6 +3191,12 @@ export const finishSlice = internalMutation({ const job = await ctx.db.get(a.jobId); if (!job) return { ok: false as const, reason: "job_not_found" as const }; if (job.leaseId !== a.leaseId) return { ok: false as const, reason: "lease_mismatch" as const }; + // Optional fallbacks preserve callers deployed before exact accounting. New callers pass + // explicit values, including zero, which are never replaced by the legacy estimates. + const modelCalls = persistedCallCount(a.modelCalls, 1, "modelCalls"); + const toolCalls = persistedCallCount(a.toolCalls, a.inputTokens || a.outputTokens ? 1 : 0, "toolCalls"); + const costKind: CostKind = a.costKind ?? "estimated"; + const accounting = await loadJobAccountingBaseline(ctx, a.jobId, job); const now = Date.now(); await ctx.db.insert("agentJobAttempts", clean({ jobId: a.jobId, @@ -3005,7 +3210,11 @@ export const finishSlice = internalMutation({ inputTokens: a.inputTokens, outputTokens: a.outputTokens, cachedInputTokens: a.cachedInputTokens, + cacheCreationInputTokens: a.cacheCreationInputTokens, costUsd: a.costUsd, + costKind, + modelCalls, + toolCalls, error: a.error, scheduledNextAt: a.scheduledNextAt, startedAt: now - a.ms, @@ -3023,8 +3232,14 @@ export const finishSlice = internalMutation({ status: nextStatus, leaseId: "", leaseUntil: 0, - modelCallCount: (job.modelCallCount ?? 0) + 1, - toolCallCount: (job.toolCallCount ?? 0) + (a.inputTokens || a.outputTokens ? 1 : 0), + modelCallCount: accounting.modelCallCount + modelCalls, + toolCallCount: accounting.toolCallCount + toolCalls, + inputTokens: accounting.inputTokens + a.inputTokens, + outputTokens: accounting.outputTokens + a.outputTokens, + cachedInputTokens: accounting.cachedInputTokens + (a.cachedInputTokens ?? 0), + cacheCreationInputTokens: accounting.cacheCreationInputTokens + (a.cacheCreationInputTokens ?? 0), + costUsd: accounting.costUsd + a.costUsd, + costKind: aggregateCostKind(accounting.costKind, costKind), mutationCount: (job.mutationCount ?? 0) + 1, updatedAt: now, }; @@ -3078,7 +3293,7 @@ export const finishSlice = internalMutation({ await recordOperationEvent(ctx, { jobId: a.jobId, runId: a.runId, - sequence: (job.actionSliceCount ?? 0) + (job.queryCount ?? 0) + (job.mutationCount ?? 0) + (job.modelCallCount ?? 0) + (job.toolCallCount ?? 0) + (job.schedulerHandoffCount ?? 0) + 3, + sequence: (job.actionSliceCount ?? 0) + (job.queryCount ?? 0) + (job.mutationCount ?? 0) + accounting.modelCallCount + accounting.toolCallCount + (job.schedulerHandoffCount ?? 0) + 3, kind: "checkpoint", name: "agentJobs.finishSlice", targetKind: "artifact", @@ -3096,7 +3311,7 @@ export const finishSlice = internalMutation({ await recordOperationEvent(ctx, { jobId: a.jobId, runId: a.runId, - sequence: (job.actionSliceCount ?? 0) + (job.queryCount ?? 0) + (job.mutationCount ?? 0) + (job.modelCallCount ?? 0) + (job.toolCallCount ?? 0) + (job.schedulerHandoffCount ?? 0) + 4, + sequence: (job.actionSliceCount ?? 0) + (job.queryCount ?? 0) + (job.mutationCount ?? 0) + accounting.modelCallCount + accounting.toolCallCount + (job.schedulerHandoffCount ?? 0) + 4, kind: "scheduler", name: "agentJobs.finishSlice.workflowSchedulerFallback", targetKind: "artifact", @@ -3162,7 +3377,8 @@ export const completeDeterministicBenchmarkSlice = internalMutation({ updatedAt: now, finalText: capStreamText(a.finalText, 60_000), error: "", - modelCallCount: (job.modelCallCount ?? 0) + 1, + // The deterministic HMDA executor never calls a model. + modelCallCount: job.modelCallCount ?? 0, mutationCount: (job.mutationCount ?? 0) + 1, }); return { ok: true as const }; diff --git a/convex/agentRuns.ts b/convex/agentRuns.ts index ae0cd723..29b9021d 100644 --- a/convex/agentRuns.ts +++ b/convex/agentRuns.ts @@ -4,6 +4,8 @@ import { internalMutation, internalQuery, query } from "./_generated/server"; import { actorProofV, requireActorProof } from "./lib"; import { findReusableRun } from "../src/nodeagent/core/idempotency"; +const costKindV = v.union(v.literal("exact"), v.literal("estimated")); + /** Claim a run row up-front (idempotency layer 1): a concurrent duplicate sees this in-flight row * (no stopReason yet) via byKey and bails before racing the same locks/CAS. Finished by `finish`. */ export const claim = internalMutation({ @@ -36,13 +38,16 @@ export const claimOrReuse = internalMutation({ /** Patch the claimed row with final telemetry (success or failure). */ export const finish = internalMutation({ args: { - runId: v.id("agentRuns"), model: v.string(), steps: v.number(), toolCalls: v.number(), conflictsSurvived: v.number(), - inputTokens: v.number(), outputTokens: v.number(), cachedInputTokens: v.optional(v.number()), costUsd: v.number(), ms: v.number(), exhausted: v.boolean(), + runId: v.id("agentRuns"), model: v.string(), steps: v.number(), modelCalls: v.optional(v.number()), toolCalls: v.number(), conflictsSurvived: v.number(), + inputTokens: v.number(), outputTokens: v.number(), cachedInputTokens: v.optional(v.number()), cacheCreationInputTokens: v.optional(v.number()), + costUsd: v.number(), costKind: v.optional(costKindV), ms: v.number(), exhausted: v.boolean(), stopReason: v.optional(v.string()), remainingMs: v.optional(v.number()), deadlineAt: v.optional(v.number()), handoff: v.optional(v.any()), }, handler: async (ctx, { runId, ...patch }) => { - const doc: Record = { ...patch }; - for (const key of ["remainingMs", "deadlineAt", "handoff"]) if (doc[key] === undefined) delete doc[key]; + const doc: Record = { ...patch, costKind: patch.costKind ?? "estimated" }; + for (const key of ["modelCalls", "cachedInputTokens", "cacheCreationInputTokens", "remainingMs", "deadlineAt", "handoff"]) { + if (doc[key] === undefined) delete doc[key]; + } await ctx.db.patch(runId, doc as any); return runId; }, @@ -59,13 +64,14 @@ export const record = internalMutation({ args: { jobId: v.optional(v.id("agentJobs")), roomId: v.id("rooms"), agentId: v.string(), model: v.string(), goal: v.string(), - steps: v.number(), toolCalls: v.number(), conflictsSurvived: v.number(), - inputTokens: v.number(), outputTokens: v.number(), cachedInputTokens: v.optional(v.number()), costUsd: v.number(), ms: v.number(), exhausted: v.boolean(), + steps: v.number(), modelCalls: v.optional(v.number()), toolCalls: v.number(), conflictsSurvived: v.number(), + inputTokens: v.number(), outputTokens: v.number(), cachedInputTokens: v.optional(v.number()), cacheCreationInputTokens: v.optional(v.number()), + costUsd: v.number(), costKind: v.optional(costKindV), ms: v.number(), exhausted: v.boolean(), stopReason: v.optional(v.string()), remainingMs: v.optional(v.number()), deadlineAt: v.optional(v.number()), handoff: v.optional(v.any()), }, handler: (ctx, a) => { - const doc: Record = { ...a, createdAt: Date.now() }; - for (const key of ["stopReason", "remainingMs", "deadlineAt", "handoff"]) { + const doc: Record = { ...a, costKind: a.costKind ?? "estimated", createdAt: Date.now() }; + for (const key of ["modelCalls", "cachedInputTokens", "cacheCreationInputTokens", "stopReason", "remainingMs", "deadlineAt", "handoff"]) { if (doc[key] === undefined) delete doc[key]; } return ctx.db.insert("agentRuns", doc as any); diff --git a/convex/credits.ts b/convex/credits.ts index 96fbd662..32b693a5 100644 --- a/convex/credits.ts +++ b/convex/credits.ts @@ -26,6 +26,7 @@ import { } from "../src/nodeagent/core/creditModel"; const creditModeV = v.union(v.literal("quick"), v.literal("standard"), v.literal("deep")); +const costKindV = v.union(v.literal("exact"), v.literal("estimated")); /** Reserve holds expire after this with no settle → swept + refunded (crashed-run holds). */ const RESERVATION_TTL_MS = 60 * 60 * 1000; // 1 hour const MAX_SWEEP_ROWS = 2_000; @@ -183,12 +184,15 @@ export const settle = internalMutation({ args: { roomId: v.id("rooms"), reservationKey: v.string(), + /** Legacy parameter name; `costKind` records whether this amount was metered or estimated. */ actualUsd: v.number(), + costKind: v.optional(costKindV), runId: v.optional(v.id("agentRuns")), now: v.optional(v.number()), }, - handler: async (ctx, { roomId, reservationKey, actualUsd, runId, now }) => { + handler: async (ctx, { roomId, reservationKey, actualUsd, costKind, runId, now }) => { const ts = now ?? Date.now(); + const settlementCostKind = costKind ?? "estimated"; const rows = await ctx.db .query("creditLedger") .withIndex("by_reservation", (q) => q.eq("reservationKey", reservationKey)) @@ -246,6 +250,7 @@ export const settle = internalMutation({ credits: -round2(usdToCredits(actualUsd)), usd: -round4(actualUsd), runId, + costKind: settlementCostKind, reason: overspentCredits > 0 ? "overspent" : undefined, createdAt: ts, }); @@ -267,6 +272,7 @@ export const settle = internalMutation({ settledCredits: round2(settledCredits), refundedCredits: round2(refundedCredits), overspentCredits: round2(overspentCredits), + costKind: settlementCostKind, balance: balanceView(await getRoomCredits(ctx, roomId)), }; }, @@ -339,8 +345,8 @@ export const sweepExpiredReservations = internalMutation({ if (!rc) continue; const hold = -row.credits; // positive // COST-AWARE so a crashed run can't silently refund money the LLM already billed: - // - finished run (agentRuns.costUsd > 0) → charge the ACTUAL cost, refund the remainder; - // - claimed-but-unsettled run (row exists, cost 0) → CAPTURE the hold (assume it spent — never lose money); + // - finished run (stopReason persisted by agentRuns.finish) → charge its recorded cost, including exact zero; + // - claimed-but-unsettled run (row exists, no terminal marker) → CAPTURE the hold (assume it spent — never lose money); // - no run row at all (never started) → refund the full hold. const run = await ctx.db .query("agentRuns") @@ -348,15 +354,19 @@ export const sweepExpiredReservations = internalMutation({ .order("desc") .first(); let actualUsd: number; + let costKind: "exact" | "estimated"; let resolution: "settled" | "captured" | "refunded"; - if (run && run.costUsd > 0) { + if (run?.stopReason !== undefined) { actualUsd = run.costUsd; + costKind = run.costKind ?? "estimated"; resolution = "settled"; } else if (run) { actualUsd = creditsToUsd(hold); + costKind = "estimated"; resolution = "captured"; } else { actualUsd = 0; + costKind = "exact"; resolution = "refunded"; } const actualCredits = Math.max(0, Math.min(hold, usdToCredits(actualUsd))); // capped at the hold @@ -367,7 +377,7 @@ export const sweepExpiredReservations = internalMutation({ lifetimeSpentCredits: round2(rc.lifetimeSpentCredits + actualCredits), updatedAt: ts, }); - await ctx.db.insert("creditLedger", { roomId: row.roomId, kind: "settle", mode: row.mode, reservationKey: row.reservationKey, credits: -round2(actualCredits), usd: -round4(actualUsd), runId: run?._id, reason: `swept_${resolution}`, createdAt: ts }); + await ctx.db.insert("creditLedger", { roomId: row.roomId, kind: "settle", mode: row.mode, reservationKey: row.reservationKey, credits: -round2(actualCredits), usd: -round4(actualUsd), runId: run?._id, costKind, reason: `swept_${resolution}`, createdAt: ts }); if (refund > 0) { await ctx.db.insert("creditLedger", { roomId: row.roomId, kind: "refund", mode: row.mode, reservationKey: row.reservationKey, credits: round2(refund), usd: round4(creditsToUsd(refund)), reason: "expired_reservation", createdAt: ts }); } diff --git a/convex/schema.ts b/convex/schema.ts index 50ae6082..0a2f0246 100644 --- a/convex/schema.ts +++ b/convex/schema.ts @@ -850,12 +850,15 @@ export default defineSchema({ model: v.string(), goal: v.string(), steps: v.number(), + modelCalls: v.optional(v.number()), toolCalls: v.number(), conflictsSurvived: v.number(), inputTokens: v.number(), outputTokens: v.number(), cachedInputTokens: v.optional(v.number()), + cacheCreationInputTokens: v.optional(v.number()), costUsd: v.number(), + costKind: v.optional(v.union(v.literal("exact"), v.literal("estimated"))), ms: v.number(), exhausted: v.boolean(), stopReason: v.optional(v.string()), @@ -901,6 +904,7 @@ export default defineSchema({ usd: v.number(), jobId: v.optional(v.id("agentJobs")), runId: v.optional(v.id("agentRuns")), + costKind: v.optional(v.union(v.literal("exact"), v.literal("estimated"))), reason: v.optional(v.string()), note: v.optional(v.string()), createdAt: v.number(), @@ -956,6 +960,12 @@ export default defineSchema({ mutationCount: v.optional(v.number()), modelCallCount: v.optional(v.number()), toolCallCount: v.optional(v.number()), + inputTokens: v.optional(v.number()), + outputTokens: v.optional(v.number()), + cachedInputTokens: v.optional(v.number()), + cacheCreationInputTokens: v.optional(v.number()), + costUsd: v.optional(v.number()), + costKind: v.optional(v.union(v.literal("exact"), v.literal("estimated"))), schedulerHandoffCount: v.optional(v.number()), receiptCount: v.optional(v.number()), latestRunId: v.optional(v.id("agentRuns")), @@ -984,7 +994,11 @@ export default defineSchema({ inputTokens: v.number(), outputTokens: v.number(), cachedInputTokens: v.optional(v.number()), + cacheCreationInputTokens: v.optional(v.number()), costUsd: v.number(), + costKind: v.optional(v.union(v.literal("exact"), v.literal("estimated"))), + modelCalls: v.optional(v.number()), + toolCalls: v.optional(v.number()), error: v.optional(v.string()), scheduledNextAt: v.optional(v.number()), startedAt: v.number(), diff --git a/e2e/benchmark-ui-spreadsheetbench-generic.spec.ts b/e2e/benchmark-ui-spreadsheetbench-generic.spec.ts index 63bb7254..6d410e90 100644 --- a/e2e/benchmark-ui-spreadsheetbench-generic.spec.ts +++ b/e2e/benchmark-ui-spreadsheetbench-generic.spec.ts @@ -4,6 +4,7 @@ import { basename, dirname, join, relative, resolve } from "node:path"; import ExcelJS from "exceljs"; import { scoreSpreadsheetBenchWorkbook } from "../src/eval/spreadsheetBenchScorer"; import type { SpreadsheetBenchTrack } from "../src/eval/spreadsheetBenchAdapter"; +import { getProviderForModel } from "../src/nodeagent/models/modelCatalog"; type AgentTaskManifest = { schema: 1; @@ -39,7 +40,9 @@ type CaseResult = { type AgentUiEvidence = { routeText: string; + resolvedModel: string; approvalPolicy: string; + attemptBudget: 12; mutationCount: number; receiptCount: number; receiptText: string; @@ -66,7 +69,6 @@ const PROOF_PATH = process.env.SPREADSHEETBENCH_LIVE_PROOF_PATH test.describe(`${TRACK} generic prod-browser adapter`, () => { test("uploads staged workbook cases, runs NodeAgent in a fresh live room, exports, and scores", async ({ page }, testInfo) => { if (!TASK_ID) throw new Error("SPREADSHEETBENCH_TASK_ID is required."); - test.setTimeout(Math.max(20 * 60_000, (CASE_LIMIT ?? 1) * (AGENT_TIMEOUT_MS + 3 * 60_000))); const staged = loadStagedTask(STAGE_ROOT, TASK_ID); expect(staged.agent.track).toBe(TRACK); @@ -75,6 +77,7 @@ test.describe(`${TRACK} generic prod-browser adapter`, () => { expect(staged.evaluator.goldFiles.length, "staged task must include evaluator-only gold workbooks").toBeGreaterThan(0); const caseCount = Math.min(staged.agent.inputFiles.length, staged.evaluator.goldFiles.length, CASE_LIMIT ?? Number.POSITIVE_INFINITY); + test.setTimeout(Math.max(20 * 60_000, caseCount * (AGENT_TIMEOUT_MS + 3 * 60_000))); const pageErrors: string[] = []; const consoleProblems: string[] = []; page.on("pageerror", (error) => pageErrors.push(error.message)); @@ -85,21 +88,33 @@ test.describe(`${TRACK} generic prod-browser adapter`, () => { }); const caseResults: CaseResult[] = []; + const cleanupState = { promptSent: false }; let runtimeFailure: string | undefined; + let cleanupFailure: string | undefined; try { for (let index = 0; index < caseCount; index += 1) { - const inputFile = staged.agent.inputFiles[index]; - const goldFile = staged.evaluator.goldFiles[index]; - if (!inputFile || !goldFile) throw new Error(`Missing case pair ${index} for ${TASK_ID}`); - const result = await runWorkbookCase(page, testInfo, staged, inputFile, goldFile, index); - caseResults.push(result); + cleanupState.promptSent = false; + try { + const inputFile = staged.agent.inputFiles[index]; + const goldFile = staged.evaluator.goldFiles[index]; + if (!inputFile || !goldFile) throw new Error(`Missing case pair ${index} for ${TASK_ID}`); + const result = await runWorkbookCase(page, testInfo, staged, inputFile, goldFile, index, cleanupState); + caseResults.push(result); + } finally { + const caseCleanupFailure = await cancelActiveJob(page, cleanupState.promptSent); + if (caseCleanupFailure && !cleanupFailure) cleanupFailure = `case ${index}: ${caseCleanupFailure}`; + } + if (cleanupFailure) break; } } catch (error) { runtimeFailure = error instanceof Error ? error.message : String(error); + } finally { + const finalCleanupFailure = await cancelActiveJob(page, cleanupState.promptSent); + if (finalCleanupFailure && !cleanupFailure) cleanupFailure = finalCleanupFailure; } const failedCases = caseResults.filter((result) => !result.passed); - const passed = !runtimeFailure && failedCases.length === 0 && pageErrors.length === 0 && consoleProblems.length === 0; + const passed = !runtimeFailure && !cleanupFailure && failedCases.length === 0 && pageErrors.length === 0 && consoleProblems.length === 0; writeProof({ schema: "proofloop-spreadsheetbench-prod-browser-receipt-v1", generatedAt: new Date().toISOString(), @@ -136,6 +151,7 @@ test.describe(`${TRACK} generic prod-browser adapter`, () => { })), failures: [ ...(runtimeFailure ? [`runtime: ${runtimeFailure}`] : []), + ...(cleanupFailure ? [`cleanup: ${cleanupFailure}`] : []), ...failedCases.map((result) => `case ${result.caseIndex}: score did not pass (${result.score.totals.mismatches} mismatch(es))`), ...pageErrors.map((error) => `pageerror: ${error}`), ...consoleProblems, @@ -148,7 +164,8 @@ test.describe(`${TRACK} generic prod-browser adapter`, () => { ...(caseResults.length > 0 ? [ "deliverable_export_download", "artifact_reopen_validation", - "official_scorer_handoff", + "local_immutable_scorer_handoff", + "bounded_12_attempt_job", "resolved_model_route_visible", "inspect_preflight_managed_write_postverify_visible", "mutation_receipt_visible", @@ -159,6 +176,7 @@ test.describe(`${TRACK} generic prod-browser adapter`, () => { }); if (runtimeFailure) throw new Error(runtimeFailure); + if (cleanupFailure) throw new Error(cleanupFailure); expect(pageErrors, "browser page errors").toEqual([]); expect(consoleProblems, "console warnings/errors").toEqual([]); expect(failedCases.map((result) => `${result.caseIndex}:${result.score.totals.mismatches}`), "all workbook cases must score cleanly").toEqual([]); @@ -172,6 +190,7 @@ async function runWorkbookCase( inputFile: string, goldFile: string, caseIndex: number, + cleanupState: { promptSent: boolean }, ): Promise { await createFreshRoom(page); await selectAgentRoute(page); @@ -188,7 +207,7 @@ async function runWorkbookCase( "Edit the workbook itself. Do not only explain the answer in chat.", "Preserve the workbook structure unless the task explicitly asks for a new layout.", `When the workbook is ready, include this exact phrase in your final answer: "${expectedPhrase}".`, - ].join("\n"), expectedPhrase); + ].join("\n"), expectedPhrase, cleanupState); const downloadPath = await exportActiveWorkbook(page, testInfo.outputPath(`spreadsheetbench-${sanitize(staged.agent.taskId)}-${caseIndex + 1}.xlsx`)); await testInfo.attach(`spreadsheetbench-case-${caseIndex + 1}-workbook`, { @@ -278,7 +297,12 @@ async function openUploadedWorkbook(page: Page, filename: string): Promise await expect(page.getByTestId("sheet-grid").or(page.locator(".r-grid")).first()).toBeVisible({ timeout: 90_000 }); } -async function invokeNodeAgent(page: Page, prompt: string, expectedPhrase: string): Promise { +async function invokeNodeAgent( + page: Page, + prompt: string, + expectedPhrase: string, + cleanupState: { promptSent: boolean }, +): Promise { const composer = page.locator('textarea[data-testid="chat-composer"]').first(); await expect(composer).toBeVisible({ timeout: 30_000 }); const agentMessages = page.locator('[data-testid="chat-message"].agent'); @@ -287,16 +311,15 @@ async function invokeNodeAgent(page: Page, prompt: string, expectedPhrase: strin const streamCountBefore = await streams.count().catch(() => 0); await composer.fill(prompt); await page.getByTestId("chat-send").click({ timeout: 30_000 }); + cleanupState.promptSent = true; await expect(page.getByTestId("chat-message").filter({ hasText: prompt.slice(0, 80) }).last()) .toBeVisible({ timeout: 30_000 }); await expect.poll(async () => { - const status = await quickText(page.getByTestId("job-status").first(), 500); - const agentMessageCount = await agentMessages.count().catch(() => 0); - const streamCount = await streams.count().catch(() => 0); - return /queued|running/i.test(status) || agentMessageCount > agentMessageCountBefore || streamCount > streamCountBefore ? 1 : 0; + const status = await reachableJobStatus(page, 500); + return status && hasKnownJobStatus(status) ? 1 : 0; }, { - message: "a fresh NodeAgent job or stream must appear after the prompt is sent", + message: "a fresh durable NodeAgent job status must remain reachable after the prompt is sent", timeout: 90_000, intervals: [1000, 2000, 5000], }).toBe(1); @@ -310,9 +333,12 @@ async function invokeNodeAgent(page: Page, prompt: string, expectedPhrase: strin let lastText = ""; let lastDetailText = ""; let routeText = ""; + let resolvedModel = ""; let sawFreshAgentOutput = false; + let sawBoundedAttemptBudget = false; while (Date.now() < deadline) { - const status = await quickText(page.getByTestId("job-status").first(), 1000); + const status = await reachableJobStatus(page, 1000); + if (!status) throw new Error("NodeAgent status became unreachable after the prompt was sent."); const latestAgentMessage = await quickText(agentMessages.last(), 1000); const latestStream = await quickText(streams.last(), 1000); const agentMessageCount = await agentMessages.count().catch(() => 0); @@ -322,16 +348,27 @@ async function invokeNodeAgent(page: Page, prompt: string, expectedPhrase: strin || streamCount > streamCountBefore || latestStream.length > 0; lastText = `${status}\n${latestAgentMessage}\n${latestStream}`.slice(-2000); + sawBoundedAttemptBudget = sawBoundedAttemptBudget || /\b\d+\s*\/\s*12\b/.test(status); const detailText = await quickText(page.getByTestId("job-detail").first(), 500); if (detailText) lastDetailText = detailText; - const route = page.locator(".r-job-route").first(); - const routeTitle = await route.getAttribute("title", { timeout: 500 }).catch(() => ""); - const visibleRoute = `${await quickText(route, 500)} ${routeTitle ?? ""}`.trim(); - if (visibleRoute) routeText = visibleRoute; - if (sawFreshAgentOutput && /\b(failed|blocked|cancelled)\b/i.test(status)) throw new Error(`NodeAgent failed: ${lastText}`); + const currentRoute = await readAgentRoute(page); + if (currentRoute.routeText) routeText = currentRoute.routeText; + if (currentRoute.resolvedModel) resolvedModel = currentRoute.resolvedModel; + if (/\b(failed|blocked|cancelled)\b/i.test(status)) throw new Error(`NodeAgent failed: ${lastText}`); if (new RegExp(escapeRegex(expectedPhrase), "i").test(`${latestAgentMessage}\n${latestStream}`) || (sawFreshAgentOutput && /\b(completed|done)\b/i.test(status))) { + expect(sawBoundedAttemptBudget, "the live durable job must expose the 12-attempt production ceiling").toBe(true); + await expect.poll(async () => { + const acceptedRoute = await readAgentRoute(page, 1000); + if (acceptedRoute.routeText) routeText = acceptedRoute.routeText; + if (acceptedRoute.resolvedModel) resolvedModel = acceptedRoute.resolvedModel; + return resolvedModel.length > 0; + }, { + message: "the completed job must expose its accepted/resolved model route", + timeout: 60_000, + intervals: [500, 1000, 2000], + }).toBe(true); await expect.poll(async () => page.locator(".r-cell.locked").count(), { timeout: 60_000 }).toBe(0); - return collectAgentEvidence(page, { detailText: lastDetailText, routeText }); + return collectAgentEvidence(page, { detailText: lastDetailText, routeText, resolvedModel }); } await page.waitForTimeout(2000); } @@ -340,7 +377,7 @@ async function invokeNodeAgent(page: Page, prompt: string, expectedPhrase: strin async function collectAgentEvidence( page: Page, - observed: { detailText: string; routeText: string }, + observed: { detailText: string; routeText: string; resolvedModel: string }, ): Promise { const stream = page.getByTestId("agent-unified-stream").last(); await expect(stream).toBeVisible({ timeout: 30_000 }); @@ -364,7 +401,10 @@ async function collectAgentEvidence( const verificationPayloads = await verifications.locator(".r-agent-part-payload").allTextContents(); expect(verificationPayloads.some(hasPassedPostWriteVerificationReceipt), "a verify_workbook result must prove a passed post-write phase").toBe(true); - expect(observed.routeText, "the requested and resolved model route must remain visible").toMatch(/openrouter|anthropic|google|openai|groq|mistral|cohere|nvidia|qwen/i); + expect(observed.resolvedModel, "the accepted/resolved model route must remain reachable").not.toBe(""); + expect(observed.routeText, "the requested and resolved model route must remain visible").toContain(observed.resolvedModel); + expect(getProviderForModel(observed.resolvedModel), `the accepted/resolved model route must be valid (${observed.resolvedModel || "missing"})`) + .not.toBeNull(); const detailToggle = page.getByTestId("job-detail-toggle").first(); const detail = page.getByTestId("job-detail").first(); const approvalPolicy = detail.getByTestId("job-approval-policy"); @@ -402,7 +442,9 @@ async function collectAgentEvidence( return { routeText: observed.routeText, + resolvedModel: observed.resolvedModel, approvalPolicy: "auto_commit_safe", + attemptBudget: 12, mutationCount, receiptCount, receiptText, @@ -411,6 +453,27 @@ async function collectAgentEvidence( }; } +async function cancelActiveJob(page: Page, promptSent: boolean): Promise { + const status = await reachableJobStatus(page, 1000); + if (!status) { + return promptSent ? "NodeAgent status was unreachable after the prompt was sent; cleanup cannot prove a terminal state." : undefined; + } + if (isTerminalJobStatus(status)) return undefined; + const cancel = page.getByTestId("job-cancel").first(); + if (!(await cancel.isVisible({ timeout: 2000 }).catch(() => false))) { + return `NodeAgent remained nonterminal without a reachable cancel control (${status}).`; + } + await cancel.click({ timeout: 10_000 }).catch(() => undefined); + const terminal = await expect.poll( + async () => { + const nextStatus = await reachableJobStatus(page, 1000); + return nextStatus ? isTerminalJobStatus(nextStatus) : false; + }, + { timeout: 30_000, intervals: [500, 1000, 2000] }, + ).toBe(true).then(() => true).catch(() => false); + return terminal ? undefined : `Timed out while cancelling the nonterminal NodeAgent job (${status}).`; +} + function hasPassedPostWriteVerificationReceipt(payload: string): boolean { try { const parsed = JSON.parse(payload) as { @@ -539,6 +602,38 @@ function escapeRegex(value: string): string { return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); } +function resolvedModelFromRoute(visibleText: string, title: string): string | undefined { + const titleMatch = title.match(/\battempt\s+\d+\s*:\s*([^\u00b7|]+)/i); + const titleModel = titleMatch?.[1]?.trim(); + if (titleModel) return titleModel; + const visibleModel = visibleText.split(/\s+\u00b7\s+/)[1]?.trim(); + return visibleModel && getProviderForModel(visibleModel) ? visibleModel : undefined; +} + +function hasKnownJobStatus(status: string): boolean { + return /\b(completed|failed|blocked|cancelled|queued|running|retrying|handoff|paused)\b/i.test(status); +} + +function isTerminalJobStatus(status: string): boolean { + return /\b(completed|failed|blocked|cancelled)\b/i.test(status); +} + +async function reachableJobStatus(page: Page, timeout: number): Promise { + const status = page.getByTestId("job-status").first(); + if (!(await status.isVisible({ timeout }).catch(() => false))) return undefined; + return (await quickText(status, timeout)).trim() || undefined; +} + +async function readAgentRoute(page: Page, timeout = 500): Promise<{ routeText: string; resolvedModel?: string }> { + const route = page.locator(".r-job-route").first(); + const title = await route.getAttribute("title", { timeout }).catch(() => ""); + const visibleText = (await quickText(route, timeout)).trim(); + return { + routeText: [visibleText, title].filter(Boolean).join(" | "), + resolvedModel: resolvedModelFromRoute(visibleText, title ?? ""), + }; +} + async function quickText(locator: ReturnType, timeout = 250): Promise { return ((await locator.textContent({ timeout }).catch(() => "")) ?? ""); } diff --git a/src/app/store.tsx b/src/app/store.tsx index a748d959..7ce8c706 100644 --- a/src/app/store.tsx +++ b/src/app/store.tsx @@ -48,7 +48,26 @@ const VARIANCE: Record = { r_rev: "+24%", r_cogs: "+27.5%", r_gp export type EditFeedback = { ok: boolean; reason?: string; version?: number }; type UndoEntry = { roomId: string; op: ChangeOp }; -export type AgentRunTelemetry = { model: string; steps: number; toolCalls: number; inputTokens: number; outputTokens: number; costUsd: number; ms: number }; +export type AgentCostKind = "exact" | "estimated"; +export type AgentRunTelemetry = { model: string; steps: number; toolCalls: number; inputTokens: number; outputTokens: number; costUsd: number; costKind: AgentCostKind; ms: number }; +type PersistedAgentRunTelemetry = Omit & { costKind?: AgentCostKind }; + +function persistedCostKind(costKind: AgentCostKind | undefined): AgentCostKind { + return costKind ?? "estimated"; +} + +export function projectAgentRunTelemetry(run: PersistedAgentRunTelemetry): AgentRunTelemetry { + return { + model: run.model, + steps: run.steps, + toolCalls: run.toolCalls, + inputTokens: run.inputTokens, + outputTokens: run.outputTokens, + costUsd: run.costUsd, + costKind: persistedCostKind(run.costKind), + ms: run.ms, + }; +} export type AgentJobTelemetry = { id: string; goal?: string; @@ -72,6 +91,8 @@ export type AgentJobTelemetry = { mutationCount?: number; modelCallCount?: number; toolCallCount?: number; + costUsd?: number; + costKind?: AgentCostKind; schedulerHandoffCount?: number; receiptCount?: number; createdAt?: number; @@ -83,7 +104,7 @@ type FreeJobRow = { _id: string; goal?: string; status: string; entrypoint?: string; scope?: string; runtime?: string; attempts: number; maxAttempts: number; runtimeProfile?: AgentRuntimeProfile; modelPolicy: string; approvalPolicy?: string; evidencePolicy?: string; handoff?: { reason?: string }; nextRunAt?: number; finalText?: string; error?: string; latestRunId?: string; actionSliceCount?: number; queryCount?: number; mutationCount?: number; - modelCallCount?: number; toolCallCount?: number; schedulerHandoffCount?: number; receiptCount?: number; createdAt?: number; updatedAt: number; + modelCallCount?: number; toolCallCount?: number; costUsd?: number; costKind?: AgentCostKind; schedulerHandoffCount?: number; receiptCount?: number; createdAt?: number; updatedAt: number; }; /** A job is an active "work lane" until it succeeds or is cancelled — failed/paused stay visible so the user can retry/dismiss. */ function isActiveFreeJob(status: string): boolean { @@ -113,6 +134,8 @@ function mapConvexFreeJob(j: FreeJobRow): AgentJobTelemetry { mutationCount: j.mutationCount, modelCallCount: j.modelCallCount, toolCallCount: j.toolCallCount, + costUsd: j.costUsd, + costKind: j.costUsd === undefined && j.costKind === undefined ? undefined : persistedCostKind(j.costKind), schedulerHandoffCount: j.schedulerHandoffCount, receiptCount: j.receiptCount, createdAt: j.createdAt, @@ -128,9 +151,27 @@ export type AgentJobAttemptTelemetry = { inputTokens: number; outputTokens: number; costUsd: number; + costKind: AgentCostKind; error?: string; scheduledNextAt?: number; }; +type PersistedAgentJobAttemptTelemetry = Omit & { costKind?: AgentCostKind }; + +export function projectAgentJobAttemptTelemetry(attempt: PersistedAgentJobAttemptTelemetry): AgentJobAttemptTelemetry { + return { + attempt: attempt.attempt, + status: attempt.status, + resolvedModel: attempt.resolvedModel, + stopReason: attempt.stopReason, + ms: attempt.ms, + inputTokens: attempt.inputTokens, + outputTokens: attempt.outputTokens, + costUsd: attempt.costUsd, + costKind: persistedCostKind(attempt.costKind), + error: attempt.error, + scheduledNextAt: attempt.scheduledNextAt, + }; +} export type AgentJobDetailTelemetry = { operations: Array<{ sequence: number; kind: string; name: string; status: string; countDelta?: number; targetKind?: string; targetId?: string; affectedIds?: string[] }>; streamEvents: PersistedAgentStreamEvent[]; @@ -168,6 +209,7 @@ export type AgentModelSelection = | { mode: "top_paid" } | { mode: "specific"; modelPolicy: string }; export type AgentRuntimeProfile = "benchmark_completion"; +const PUBLIC_BENCHMARK_COMPLETION_MAX_ATTEMPTS = 12; export type AgentAskInput = { goal: string; references?: ArtifactRef[]; @@ -251,7 +293,9 @@ function browserNodeAgentRuntimeProfile(): AgentRuntimeProfile | undefined { } function maxAttemptsForRuntimeProfile(runtimeProfile: AgentRuntimeProfile | undefined, requested?: number): number | undefined { - if (runtimeProfile === "benchmark_completion") return Math.max(requested ?? 1000, 1000); + if (runtimeProfile === "benchmark_completion") { + return Math.max(1, Math.min(requested ?? PUBLIC_BENCHMARK_COMPLETION_MAX_ATTEMPTS, PUBLIC_BENCHMARK_COMPLETION_MAX_ATTEMPTS)); + } return requested; } @@ -684,7 +728,7 @@ export function EngineStoreProvider({ roomId, children }: { roomId: string; me: memLongJobCurrentRef.current = job; memLongJobTasksRef.current.set(id, { goal, references, cancelledAttempts: new Set() }); setMemLongJob(job); - setMemLongJobAttempts([{ attempt: 1, status: "running", resolvedModel: "scripted/free-auto", stopReason: "in_progress", ms: 0, inputTokens: 0, outputTokens: 0, costUsd: 0 }]); + setMemLongJobAttempts([{ attempt: 1, status: "running", resolvedModel: "scripted/free-auto", stopReason: "in_progress", ms: 0, inputTokens: 0, outputTokens: 0, costUsd: 0, costKind: "exact" }]); setMemLongJobDetail(memoryFreeJobDetail(goal, "running")); return id; }, []); @@ -1169,7 +1213,7 @@ export function EngineStoreProvider({ roomId, children }: { roomId: string; me: setMemLongJob(retried); if (task) task.cancelledAttempts.delete(nextAttempt); void runMemoryFreeJob(jobId, nextAttempt); - setMemLongJobAttempts((cur) => [...cur, { attempt: nextAttempt, status: "running", resolvedModel: "scripted/free-auto", stopReason: "retrying", ms: 0, inputTokens: 0, outputTokens: 0, costUsd: 0 }]); + setMemLongJobAttempts((cur) => [...cur, { attempt: nextAttempt, status: "running", resolvedModel: "scripted/free-auto", stopReason: "retrying", ms: 0, inputTokens: 0, outputTokens: 0, costUsd: 0, costKind: "exact" }]); setMemLongJobDetail((cur) => cur ? memoryFreeJobDetail(cur.reasoningFrames[0]?.goal ?? "Memory free-auto job", "running") : cur); return { ok: true }; }, @@ -2178,37 +2222,15 @@ export function ConvexStoreProvider({ roomId, me, proof, children }: { roomId: s }); }, lastRun: () => { - const r = (runs as unknown as AgentRunTelemetry[])[0]; - return r ? { model: r.model, steps: r.steps, toolCalls: r.toolCalls, inputTokens: r.inputTokens, outputTokens: r.outputTokens, costUsd: r.costUsd, ms: r.ms } : null; + const r = (runs as unknown as PersistedAgentRunTelemetry[])[0]; + return r ? projectAgentRunTelemetry(r) : null; }, lastLongFreeJob: () => { const j = (jobs as FreeJobRow[])[0]; return j ? mapConvexFreeJob(j) : null; }, activeLongFreeJobs: () => (jobs as FreeJobRow[]).filter((j) => isActiveFreeJob(j.status)).map(mapConvexFreeJob), - lastLongFreeJobAttempts: () => (jobAttempts as Array<{ - attempt: number; - status: string; - resolvedModel: string; - stopReason: string; - ms: number; - inputTokens: number; - outputTokens: number; - costUsd: number; - error?: string; - scheduledNextAt?: number; - }>).map((a) => ({ - attempt: a.attempt, - status: a.status, - resolvedModel: a.resolvedModel, - stopReason: a.stopReason, - ms: a.ms, - inputTokens: a.inputTokens, - outputTokens: a.outputTokens, - costUsd: a.costUsd, - error: a.error, - scheduledNextAt: a.scheduledNextAt, - })), + lastLongFreeJobAttempts: () => (jobAttempts as PersistedAgentJobAttemptTelemetry[]).map(projectAgentJobAttemptTelemetry), lastLongFreeJobDetail: () => { if (!jobDetail) return null; const d = jobDetail as { diff --git a/src/nodeagent/core/proofloopSupervisor.ts b/src/nodeagent/core/proofloopSupervisor.ts index 32fc2374..76cafd7a 100644 --- a/src/nodeagent/core/proofloopSupervisor.ts +++ b/src/nodeagent/core/proofloopSupervisor.ts @@ -4,6 +4,7 @@ import type { AgentMessage, AgentResult } from "./types"; export const PROOFLOOP_VERIFIER_REPAIR_PREFIX = "PROOFLOOP VERIFIER REPAIR:"; export const PROOFLOOP_NO_WRITE_SPEND_BUDGET = "proofloop_no_write_spend_budget"; export const PROOFLOOP_NO_PROGRESS_AFTER_REPAIR = "proofloop_no_progress_after_repair"; +export const PROOFLOOP_REPEATED_WORKFLOW_BLOCK = "proofloop_repeated_workflow_block"; export type ProofloopSupervisorDecision = | { kind: "none" } @@ -19,14 +20,42 @@ type ProofloopSupervisorInput = { }; const WRITE_TOOLS = new Set(DEFAULT_WRITE_TOOLS); +const VERIFIED_WORKFLOW_BLOCK_LIMIT = 4; export function proofloopSupervisorDecision(input: ProofloopSupervisorInput): ProofloopSupervisorDecision { if (input.runtimeProfile !== "benchmark_completion") return { kind: "none" }; - if (input.result.stopReason !== "spend_budget") return { kind: "none" }; if (!goalRequiresMaterialWrite(input.goal)) return { kind: "none" }; - if (hasRoomWriteAttempt(input.result)) return { kind: "none" }; + if (hasSuccessfulRoomWriteReceipt(input.result)) return { kind: "none" }; + + const workflowBlocks = verifiedWorkbookWorkflowBlockCount(input.result); + if (workflowBlocks >= VERIFIED_WORKFLOW_BLOCK_LIMIT) { + const reason = `Benchmark completion accumulated ${workflowBlocks} verified-workbook workflow blocks without a successful room-write receipt.`; + return boundedRepairDecision(input, { + reason, + terminalError: PROOFLOOP_REPEATED_WORKFLOW_BLOCK, + prompt: [ + `${PROOFLOOP_VERIFIER_REPAIR_PREFIX} The prior slice repeatedly failed the verified workbook commit boundary.`, + "Do not retranscribe the complete approved plan.", + "If preflight already passed, call write_locked_cells once with the approved artifact id and one unchanged approved operation; the runtime will bind that explicit commit attempt to the complete preflight-approved plan.", + "Do not change targets, formulas, values, cached results, or number formats. If no plan passed preflight, inspect and preflight one corrected complete plan first.", + ].join(" "), + }); + } + + if (input.result.stopReason !== "spend_budget") return { kind: "none" }; const reason = "Benchmark completion hit spend_budget without any room-write tool receipt for a required-write goal."; + return boundedRepairDecision(input, { + reason, + terminalError: PROOFLOOP_NO_WRITE_SPEND_BUDGET, + prompt: buildRepairPrompt(input), + }); +} + +function boundedRepairDecision( + input: ProofloopSupervisorInput, + args: { reason: string; terminalError: string; prompt: string }, +): ProofloopSupervisorDecision { const repairAlreadyIssued = hasProofloopRepairPrompt(input.result.messages); const noAttemptsRemaining = input.attempt >= input.maxAttempts; const crossedRepairLimit = input.attempt >= 2; @@ -34,16 +63,16 @@ export function proofloopSupervisorDecision(input: ProofloopSupervisorInput): Pr return { kind: "terminal_failure", reason: repairAlreadyIssued - ? `${reason} A verifier repair prompt was already issued, so the job is failing instead of looping.` - : `${reason} No bounded repair attempt remains, so the job is failing instead of looping.`, - error: repairAlreadyIssued ? PROOFLOOP_NO_PROGRESS_AFTER_REPAIR : PROOFLOOP_NO_WRITE_SPEND_BUDGET, + ? `${args.reason} A verifier repair prompt was already issued, so the job is failing instead of looping.` + : `${args.reason} No bounded repair attempt remains, so the job is failing instead of looping.`, + error: repairAlreadyIssued ? PROOFLOOP_NO_PROGRESS_AFTER_REPAIR : args.terminalError, }; } return { kind: "repair", - reason, - prompt: buildRepairPrompt(input), + reason: args.reason, + prompt: args.prompt, }; } @@ -64,6 +93,47 @@ export function hasRoomWriteAttempt(result: Pick WRITE_TOOLS.has(call.tool)))); } +export function hasSuccessfulRoomWriteReceipt(result: Pick): boolean { + if (result.trace.some((event) => WRITE_TOOLS.has(event.tool) && successfulWriteResult(event.result))) return true; + return result.messages.some((message) => { + if (message.role !== "tool" || !message.toolName || !WRITE_TOOLS.has(message.toolName)) return false; + return successfulWriteResult(parseToolResult(message.content)); + }); +} + +export function verifiedWorkbookWorkflowBlockCount(result: Pick): number { + const traceCount = result.trace.filter((event) => + WRITE_TOOLS.has(event.tool) && verifiedWorkbookBlockResult(event.result)).length; + const messageCount = result.messages.filter((message) => + message.role === "tool" + && !!message.toolName + && WRITE_TOOLS.has(message.toolName) + && verifiedWorkbookBlockResult(parseToolResult(message.content))).length; + return Math.max(traceCount, messageCount); +} + +function successfulWriteResult(value: unknown): boolean { + if (!value || typeof value !== "object" || Array.isArray(value)) return false; + const result = value as Record; + return result.ok !== false && typeof result.error !== "string"; +} + +function verifiedWorkbookBlockResult(value: unknown): boolean { + if (!value || typeof value !== "object" || Array.isArray(value)) return false; + const result = value as Record; + return result.error === "tool_blocked" + && typeof result.reason === "string" + && result.reason.includes("verified_workbook_workflow:"); +} + +function parseToolResult(content: string): unknown { + try { + return JSON.parse(content); + } catch { + return undefined; + } +} + function goalRequiresMaterialWrite(goal: string): boolean { if (goalForbidsMaterialWrites(goal)) return false; return /\b(write|fill|edit|update|set|create|delete|recompute|commit|apply)\b/i.test(goal); diff --git a/src/nodeagent/core/runtime.ts b/src/nodeagent/core/runtime.ts index 441d4908..b59dcc57 100644 --- a/src/nodeagent/core/runtime.ts +++ b/src/nodeagent/core/runtime.ts @@ -4,7 +4,7 @@ * becomes a re-read-and-retry instead of a silent overwrite. */ -import type { AgentModel, AgentTool, RoomTools, AgentResult, AgentMessage, AgentTraceEvent, AgentStopReason, AgentHandoff, ToolCall, AgentStep, ToolArgumentErrorResult } from "./types"; +import type { AgentModel, AgentTool, RoomTools, AgentResult, AgentMessage, AgentTraceEvent, AgentStopReason, AgentHandoff, ToolCall, AgentStep, ToolArgumentErrorResult, TokenUsage } from "./types"; import { toolStreamEvidenceMetadata, type AgentStreamEventDraft } from "./stream"; import type { StepJournal } from "./journal"; import { checkSpendCeiling, type SpendLimits } from "../guardrails/gateway"; @@ -39,41 +39,86 @@ export class AgentRunError extends Error { } const DEFAULT_RESERVE_MS = 15_000; +const ABORT_SETTLEMENT_GRACE_MS = 250; const BTB_PACKAGE_NUDGE = "BTB PACKAGE CONTRACT:"; const BTB_PACKAGE_ONLY_AFTER_NUDGES = 1; const BTB_READ_TOOL_TURN_LIMIT = 24; export const TOOL_REQUIRED_NO_CALL_MARKER = "tool_required_no_call"; export const TOOL_REQUIRED_NO_CALL_TERMINAL_MARKER = "tool_required_no_call_terminal"; const TOOL_REQUIRED_NO_CALL_TERMINAL_AFTER = 4; +const DETERMINISTIC_TOOL_FAILURE_TERMINAL_AFTER = 4; function isAbortLike(error: unknown): boolean { return error instanceof Error && (error.name === "AbortError" || /aborted|abort/i.test(error.message)); } -function abortError(): Error { +function abortError(): Error & { usage: TokenUsage } { const error = new Error("model call aborted by the NodeAgent time budget"); error.name = "AbortError"; - return error; + return Object.assign(error, { + // The provider adapter did not settle in the bounded grace period, so the + // only honest accounting is unknown/estimated. Do not emit an exact zero. + usage: { + inputTokens: 0, + outputTokens: 0, + modelCalls: 0, + costKind: "estimated" as const, + }, + }); +} + +function providerFailureUsage(error: unknown): TokenUsage | undefined { + if (!error || typeof error !== "object") return undefined; + const usage = (error as { usage?: unknown }).usage; + if (!usage || typeof usage !== "object") return undefined; + const value = usage as Partial; + if (!Number.isFinite(value.inputTokens) || !Number.isFinite(value.outputTokens)) return undefined; + return { + inputTokens: Math.max(0, value.inputTokens ?? 0), + outputTokens: Math.max(0, value.outputTokens ?? 0), + cachedInputTokens: Math.max(0, value.cachedInputTokens ?? 0), + cacheCreationInputTokens: Math.max(0, value.cacheCreationInputTokens ?? 0), + modelCalls: Math.max(0, value.modelCalls ?? 0), + ...(Number.isFinite(value.costUsd) ? { costUsd: Math.max(0, value.costUsd ?? 0) } : {}), + ...(value.costKind === "exact" || value.costKind === "estimated" ? { costKind: value.costKind } : {}), + }; +} + +function requiredNoToolMissCount(messages: AgentMessage[]): number { + let max = 0; + const pattern = new RegExp(`${TOOL_REQUIRED_NO_CALL_MARKER}\\s+(\\d+)\\/${TOOL_REQUIRED_NO_CALL_TERMINAL_AFTER}`, "i"); + for (const message of messages) { + if (message.role !== "user") continue; + const count = Number(message.content.match(pattern)?.[1] ?? 0); + if (Number.isFinite(count)) max = Math.max(max, count); + } + return max; } async function settleWithAbort(promise: Promise, signal?: AbortSignal): Promise { if (!signal) return promise; - if (signal.aborted) throw abortError(); return new Promise((resolve, reject) => { + let graceTimer: ReturnType | undefined; + let settled = false; + const finish = (complete: () => void) => { + if (settled) return; + settled = true; + signal.removeEventListener("abort", onAbort); + if (graceTimer !== undefined) clearTimeout(graceTimer); + complete(); + }; const onAbort = () => { signal.removeEventListener("abort", onAbort); - reject(abortError()); + // Fetch/stream adapters attach partial usage while unwinding an abort. + // Wait briefly for that richer failure before falling back to an honest + // estimated interruption; the grace remains inside the persistence reserve. + graceTimer = setTimeout(() => finish(() => reject(abortError())), ABORT_SETTLEMENT_GRACE_MS); }; - signal.addEventListener("abort", onAbort, { once: true }); + if (signal.aborted) onAbort(); + else signal.addEventListener("abort", onAbort, { once: true }); promise.then( - (value) => { - signal.removeEventListener("abort", onAbort); - resolve(value); - }, - (error) => { - signal.removeEventListener("abort", onAbort); - reject(error); - }, + (value) => finish(() => resolve(value)), + (error) => finish(() => reject(error)), ); }); } @@ -85,6 +130,18 @@ function toolResultFailed(result: unknown): boolean { return object.ok === false || typeof object.error === "string"; } +function deterministicToolFailureKey(toolName: string, result: unknown): string | undefined { + if (!result || typeof result !== "object" || Array.isArray(result)) return undefined; + const value = result as Record; + const error = typeof value.error === "string" ? value.error : undefined; + if (error !== "tool_blocked" && error !== "tool_argument_error") return undefined; + const failureKind = typeof value.failureKind === "string" ? value.failureKind : "unknown"; + const reason = typeof value.reason === "string" ? value.reason : ""; + const workflowStage = reason.match(/verified_workbook_workflow:([^:]+):/i)?.[1]; + const reasonKey = workflowStage ? `verified_workbook_workflow:${workflowStage}` : reason.slice(0, 300); + return JSON.stringify([toolName, error, failureKind, reasonKey]); +} + function toolArgumentErrorResult(toolName: string, issues: Array<{ path: PropertyKey[]; code: string; message: string }>): ToolArgumentErrorResult { const normalized = issues.map((issue) => ({ path: issue.path.map(String).join("."), @@ -311,10 +368,6 @@ function countUserNotes(messages: AgentMessage[], prefix: string): number { return messages.filter((message) => message.role === "user" && message.content?.startsWith(prefix)).length; } -function countUserNotesContaining(messages: AgentMessage[], text: string): number { - return messages.filter((message) => message.role === "user" && message.content?.includes(text)).length; -} - function countToolResults(messages: AgentMessage[], toolNames: Set, outcome: "success" | "failure"): number { let count = 0; for (const message of messages) { @@ -431,7 +484,8 @@ export async function runAgent(opts: { const messages: AgentMessage[] = []; const trace: AgentTraceEvent[] = []; let finalText = ""; - let inputTokens = 0, outputTokens = 0, modelCalls = 0, costUsd = 0, cachedInputTokens = 0; + let inputTokens = 0, outputTokens = 0, modelCalls = 0, costUsd = 0, cachedInputTokens = 0, cacheCreationInputTokens = 0; + let costKind: "exact" | "estimated" = "exact"; let attemptedSteps = 0; // P1-3: tool calls not yet executed in the current turn — preserved on an error handoff so the // resume cursor never carries unpaired assistant tool_use blocks. @@ -464,6 +518,24 @@ export async function runAgent(opts: { now, }); const shouldHandoffForTime = () => deadlineAt !== undefined && now() + reserveMs >= deadlineAt; + const recordModelUsage = (usage: TokenUsage | undefined) => { + modelCalls += usage?.modelCalls === undefined ? 1 : Math.max(0, usage.modelCalls); + if (!usage) { + costKind = "estimated"; + return; + } + inputTokens += usage.inputTokens; + outputTokens += usage.outputTokens; + cachedInputTokens += usage.cachedInputTokens ?? 0; + cacheCreationInputTokens += usage.cacheCreationInputTokens ?? 0; + if (usage.costUsd !== undefined) { + costUsd += usage.costUsd; + if (usage.costKind === "estimated") costKind = "estimated"; + return; + } + costUsd += opts.priceStep?.(model.name, usage.inputTokens, usage.outputTokens) ?? 0; + costKind = "estimated"; + }; const emitStreamEvent = (event: AgentStreamEventDraft) => { try { const result = opts.onStreamEvent?.({ createdAt: now(), ...event }); @@ -512,17 +584,33 @@ export async function runAgent(opts: { budget: budget(attempted), trace, messages, - usage: { inputTokens, outputTokens, modelCalls, cachedInputTokens }, + usage: { inputTokens, outputTokens, modelCalls, cachedInputTokens, cacheCreationInputTokens, costUsd, costKind }, + modelRouteState: model.routeState?.(), }); + const discardPendingAssistantToolCalls = (calls: ToolCall[]) => { + if (calls.length === 0) return; + const pendingIds = new Set(calls.map((call) => call.id)); + for (let index = messages.length - 1; index >= 0; index -= 1) { + const message = messages[index]; + if (message.role !== "assistant" || !message.toolCalls?.some((call) => pendingIds.has(call.id))) continue; + const retainedCalls = message.toolCalls.filter((call) => !pendingIds.has(call.id)); + if (retainedCalls.length === 0 && !message.content.trim()) messages.splice(index, 1); + else message.toolCalls = retainedCalls.length > 0 ? retainedCalls : undefined; + return; + } + }; const emitHandoff = ( step: number, reason: Exclude, attempted: number, remainingToolCalls: ToolCall[] = [], customSummary?: string, + terminalReason?: AgentHandoff["terminalReason"], ) => { - const handoff = makeHandoff(reason, attempted, remainingToolCalls); + if (terminalReason) discardPendingAssistantToolCalls(remainingToolCalls); + const handoff = makeHandoff(reason, attempted, terminalReason ? [] : remainingToolCalls); if (customSummary) handoff.summary = customSummary; + if (terminalReason) handoff.terminalReason = terminalReason; const ev: AgentTraceEvent = { step, tool: "handoff", args: { reason, deadlineAt, reserveMs }, result: handoff, ms: 0 }; trace.push(ev); opts.onTrace?.(ev); @@ -532,7 +620,7 @@ export async function runAgent(opts: { status: "skipped", title: "Agent paused", text: handoff.summary, - metadata: { reason, remainingToolCalls: remainingToolCalls.length }, + metadata: { reason, remainingToolCalls: handoff.remainingToolCalls.length, terminalReason }, }); opts.onHandoff?.(handoff); return handoff; @@ -675,6 +763,8 @@ export async function runAgent(opts: { let readNudged = false; let doneNudged = false; let requiredNoToolNudges = 0; + let deterministicFailureKey: string | undefined; + let deterministicFailureCount = 0; const managedWriteToolsAvailable = tools.some((tool) => tool.name.startsWith("write_locked_cell")); const btbPackageToolAvailable = tools.some((tool) => tool.name === "create_btb_deliverable_package"); const btbPackageTools = new Set(["create_btb_deliverable_package"]); @@ -701,7 +791,7 @@ export async function runAgent(opts: { writeCalls = countToolResults(messages, WRITE_TOOLS, "success"); lockCalls = countToolCalls(messages, new Set(["propose_lock"])); readNudged = countUserNotes(messages, "HARNESS NOTE: every tool call so far has been a read.") > 0; - requiredNoToolNudges = countUserNotesContaining(messages, TOOL_REQUIRED_NO_CALL_MARKER); + requiredNoToolNudges = requiredNoToolMissCount(messages); doneNudged = countUserNotes(messages, "HARNESS NOTE: the user asked for a write/fill/update") > 0 || countUserNotes(messages, "HARNESS NOTE: the user asked for a BTB deliverable package") > 0 || countUserNotes(messages, "HARNESS NOTE: this run cannot be complete") > 0 @@ -768,6 +858,11 @@ export async function runAgent(opts: { let out: AgentStep; if (cached) { out = cached; + // A replay makes no new provider request, but it must rematerialize the + // original request's usage in the slice result. Otherwise a crash after + // the journal commit but before the final checkpoint permanently loses + // calls, tokens, and cost from the durable run/job accounting. + recordModelUsage(cached.usage); } else { // Gateway spend ceiling — stop before a billable call once the per-run token OR dollar cap // is hit (resumable). costUsd accumulates via opts.priceStep (P0-4: previously hardcoded 0, @@ -798,6 +893,8 @@ export async function runAgent(opts: { toolChoice: requiresToolThisTurn ? "required" : "auto", }), signal.signal); } catch (error) { + const failedUsage = providerFailureUsage(error); + if (failedUsage) recordModelUsage(failedUsage); if (signal.signal?.aborted || (shouldHandoffForTime() && isAbortLike(error))) { const handoff = emitHandoff(step, "time_budget", attemptedSteps); return finish("time_budget", attemptedSteps, true, handoff); @@ -807,13 +904,9 @@ export async function runAgent(opts: { signal.cancel(); } await opts.journal?.record(step, fresh); - modelCalls++; // count + bill ONLY a real model call (a replayed step was already billed) - if (fresh.usage) { - inputTokens += fresh.usage.inputTokens; - outputTokens += fresh.usage.outputTokens; - cachedInputTokens += fresh.usage.cachedInputTokens ?? 0; - costUsd += opts.priceStep?.(model.name, fresh.usage.inputTokens, fresh.usage.outputTokens) ?? 0; - } + // Count/bill only real provider calls. A failover adapter can represent several provider + // requests in one logical model.next; replayed journal steps remain excluded. + recordModelUsage(fresh.usage); out = fresh; } if (out.text) finalText = out.text; @@ -834,7 +927,7 @@ export async function runAgent(opts: { const hasFinalText = !!(out.text?.trim() || finalText.trim()); const stillNeedsWrite = (goalRequiresWrite && writeCalls === 0) || (goalRequiresPackage && btbPackageSuccesses === 0); const noRequiredToolCall = requiresToolThisTurn && out.toolCalls.length === 0; - if (noRequiredToolCall && (doneNudged || packageOnlyMode)) { + if (noRequiredToolCall) { if (out.text) messages.push({ role: "assistant", content: out.text }); requiredNoToolNudges += 1; messages.push({ @@ -849,15 +942,24 @@ export async function runAgent(opts: { text: "Provider returned no tool call during a required-write turn; NodeAgent refreshed the instruction and preserved the trace for the next slice.", metadata: { requiredNoToolNudges, requiredAfter: TOOL_REQUIRED_NO_CALL_TERMINAL_AFTER, goalRequiresPackage, goalRequiresWrite }, }); - if (requiredNoToolNudges < TOOL_REQUIRED_NO_CALL_TERMINAL_AFTER && step < maxSteps - 1) { - continue; + if (requiredNoToolNudges < TOOL_REQUIRED_NO_CALL_TERMINAL_AFTER) { + if (step < maxSteps - 1) continue; + const handoff = emitHandoff( + step + 1, + "step_budget", + step + 1, + [], + `NodeAgent checkpointed after ${requiredNoToolNudges}/${TOOL_REQUIRED_NO_CALL_TERMINAL_AFTER} required-tool misses; the next slice must continue from this protocol state.`, + ); + return finish("step_budget", step + 1, true, handoff); } const handoff = emitHandoff( step + 1, "step_budget", step + 1, [], - `required tool call missing after ${requiredNoToolNudges} required tool-use turn${requiredNoToolNudges === 1 ? "" : "s"}; checkpointed with a narrowed instruction so the next slice can force the required ${goalRequiresPackage ? "deliverable package tool" : "write tool"}.`, + `NodeAgent stopped: required tool call missing after ${requiredNoToolNudges} required tool-use turn${requiredNoToolNudges === 1 ? "" : "s"}. Retry with a different model route or corrected provider configuration.`, + "protocol_stall", ); return finish("step_budget", step + 1, true, handoff); } @@ -956,6 +1058,36 @@ export async function runAgent(opts: { finalText = btbPackageCompletionText(goal, call.args, result) ?? finalText; } } + const failureKey = deterministicToolFailureKey(call.tool, result); + if (failureKey) { + if (failureKey === deterministicFailureKey) deterministicFailureCount += 1; + else { + deterministicFailureKey = failureKey; + deterministicFailureCount = 1; + } + } else { + deterministicFailureKey = undefined; + deterministicFailureCount = 0; + } + if (deterministicFailureCount >= DETERMINISTIC_TOOL_FAILURE_TERMINAL_AFTER) { + emitStreamEvent({ + kind: "warning", + step, + status: "failed", + title: "Repeated tool failure", + text: `NodeAgent stopped after ${deterministicFailureCount} identical deterministic tool failures.`, + metadata: { tool: call.tool, deterministicFailureCount }, + }); + const handoff = emitHandoff( + step + 1, + "step_budget", + step + 1, + pendingToolCalls, + `NodeAgent stopped after ${deterministicFailureCount} identical deterministic failures from ${call.tool}. Retry with corrected arguments, a corrected approved plan, or a different model route.`, + "protocol_stall", + ); + return finish("step_budget", step + 1, true, handoff); + } if (call.tool === "say" && !toolResultFailed(result)) { finalText = sayTextFromArgs(call.args) ?? finalText; if (pendingToolCalls.length === 0 && !goalRequiresWrite && !goalRequiresPackage) { diff --git a/src/nodeagent/core/types.ts b/src/nodeagent/core/types.ts index 43db4db8..ba21fe92 100644 --- a/src/nodeagent/core/types.ts +++ b/src/nodeagent/core/types.ts @@ -36,7 +36,20 @@ export interface ToolCall { /** Provider-specific metadata to round-trip (e.g. Gemini 3.x thought_signature, required for multi-turn tools). */ providerMetadata?: Record; } -export interface TokenUsage { inputTokens: number; outputTokens: number; /** Cached prefix-hit input tokens (provider-reported); #1 cache-health metric. */ cachedInputTokens?: number; } +export interface TokenUsage { + inputTokens: number; + outputTokens: number; + /** Cached prefix-hit input tokens (provider-reported); #1 cache-health metric. */ + cachedInputTokens?: number; + /** Provider-reported cache-write input tokens, included in inputTokens. */ + cacheCreationInputTokens?: number; + /** Exact provider requests represented by this logical model turn (for failover routes). */ + modelCalls?: number; + /** Aggregate provider cost represented by this logical model turn. */ + costUsd?: number; + /** Whether costUsd came from route-owned pricing or a conservative fallback estimate. */ + costKind?: "exact" | "estimated"; +} /** One turn of the model: optional prose + zero or more tool calls + token usage. */ export interface AgentStep { text?: string; @@ -50,8 +63,15 @@ export interface AgentStep { /* ── seam 1: the injectable model ── */ export type AgentToolChoice = "auto" | "required"; +export interface AgentModelRouteState { + preferredModelId?: string; + cooldownUntil?: Record; +} + export interface AgentModel { readonly name: string; + /** Durable routing state that callers can checkpoint between action slices. */ + routeState?(): AgentModelRouteState; next(input: { system: string; messages: AgentMessage[]; @@ -112,6 +132,8 @@ export interface AgentBudgetSnapshot { } export interface AgentHandoff { reason: Exclude; + /** Structured terminal classification for a handoff that must not be resumed automatically. */ + terminalReason?: "protocol_stall"; summary: string; nextGoal: string; remainingToolCalls: ToolCall[]; @@ -129,6 +151,7 @@ export interface AgentResult { trace: AgentTraceEvent[]; messages: AgentMessage[]; usage: TokenUsage & { modelCalls: number }; + modelRouteState?: AgentModelRouteState; } /* ── seam 2: the room-tools port (in-memory now, Convex later — SAME shape) ── */ diff --git a/src/nodeagent/guardrails/workbookWorkflow.ts b/src/nodeagent/guardrails/workbookWorkflow.ts index 7085046b..7bbb6109 100644 --- a/src/nodeagent/guardrails/workbookWorkflow.ts +++ b/src/nodeagent/guardrails/workbookWorkflow.ts @@ -16,6 +16,7 @@ const PRIMARY_ARTIFACT = "__primary_workbook__"; type WorkbookOperation = { elementId: string; target: string; + baseVersion?: number; formula?: string; value?: unknown; result?: unknown; @@ -148,7 +149,7 @@ class WorkbookWorkflowState { reason: "A passing workbook preflight has not been executed through its matching managed write.", prompt: workflowPrompt( "APPROVED_WRITE_REQUIRED", - `Execute the approved plan exactly with ${JSON.stringify(writeContract(approved))}, then call verify_workbook with afterWrite=true for the same operations.`, + `Call a managed write for ${displayArtifact(approved.artifactId)} with at least one unchanged operation, including its baseVersion when preflighted, from the approved ${approved.operations.length}-operation plan. The runtime will bind that explicit commit attempt to the complete preflight-approved plan; then call verify_workbook with afterWrite=true for the same operations.`, ), }; } @@ -202,6 +203,9 @@ class WorkbookWorkflowState { ); } if (pending.signature !== plan.signature) { + if (operationsMatchApprovedPlan(plan.operations, pending.operations, true)) { + return bindPendingVerification(call, pending); + } return blocked( "post_write_plan_mismatch", "Post-write verification must cover exactly the operations from the most recent successful managed write.", @@ -213,29 +217,54 @@ class WorkbookWorkflowState { } private guardWrite(call: ToolCall): ToolCall | BlockedTool { - const plan = this.planFromArgs(call.args, "write"); - const approved = this.approved.get(plan.artifactKey); - if (!approved) { + const requestedPlan = this.planFromArgs(call.args, "write"); + const approvedForArtifact = this.approved.get(requestedPlan.artifactKey); + const approvedPlans = [...this.approved.values()]; + if (approvedForArtifact?.signature === requestedPlan.signature) return call; + if (approvedPlans.length === 0) { return blocked( "preflight_required", "A passing verify_workbook preflight is required before any managed write for this complex workbook task.", "Call inspect_workbook, then verify_workbook with afterWrite=false and the complete operation set before retrying the managed write.", - { artifactId: plan.artifactId }, + { artifactId: requestedPlan.artifactId }, ); } - if (approved.signature !== plan.signature) { - return blocked( - "write_plan_mismatch", - "The managed write must match the most recent passing preflight plan exactly, including targets, formulas or values, cached results, and number formats.", - `Retry the approved write exactly as ${JSON.stringify(writeContract(approved))}, or submit a corrected replacement preflight first.`, - { - approvedWrite: writeContract(approved), - approvedTargets: approved.operations.map((operation) => operation.target), - requestedTargets: plan.operations.map((operation) => operation.target), - }, - ); + + // The preflight result is the write authority. Providers still have to prove intent by + // repeating at least one unchanged approved operation, but they do not have to retranscribe a + // large operation bundle perfectly. Rebinding the explicit commit attempt here keeps the + // mutation inside the already verified target/content boundary. + const bindingMatches = call.tool === "write_locked_cells" + ? approvedPlans.filter((approved) => + artifactAllowsApprovedBinding(call.args, approved.artifactId) + && rawBatchOperationsMatchApprovedSubset(call.args, approved)) + : []; + if (bindingMatches.length === 1) { + return bindApprovedBatchWrite(call, bindingMatches[0]); } - return call; + + const approved = approvedForArtifact ?? (approvedPlans.length === 1 ? approvedPlans[0] : undefined); + const approvedTargets = approved ? targetSummary(approved.operations) : undefined; + const requestedTargets = targetSummary(requestedPlan.operations); + return blocked( + "write_plan_mismatch", + "The managed write must match one passing preflight plan exactly, including artifact, targets, base versions, formulas or values, cached results, and number formats.", + approved + ? `Retry write_locked_cells with the approved artifact and at least one unchanged operation from its ${approved.operations.length}-operation plan, or submit a corrected replacement preflight first.` + : "Retry with the exact artifact and operation set from one passing preflight, or submit a corrected replacement preflight first.", + { + approvedPlanCount: approvedPlans.length, + bindingMatchCount: bindingMatches.length, + ...(approved ? { + approvedArtifactId: approved.artifactId, + approvedOperationCount: approved.operations.length, + approvedTargets: approvedTargets?.targets, + approvedTargetsOmitted: approvedTargets?.omitted, + } : {}), + requestedTargets: requestedTargets.targets, + requestedTargetsOmitted: requestedTargets.omitted, + }, + ); } private hydrate(messages: readonly AgentMessage[], strict: boolean): void { @@ -272,22 +301,30 @@ class WorkbookWorkflowState { if (call.tool === VERIFY_TOOL) { const artifactId = artifactIdFrom(args, resultRecord, this.activeArtifactId); this.activeArtifactId = artifactId; - const plan = this.planFromArgs({ ...args, artifactId }, "verify"); + const requestedPlan = this.planFromArgs({ ...args, artifactId }, "verify"); const preflight = resultRecord?.phase === "preflight" || args.afterWrite === false; const passed = resultRecord?.status === "passed" && resultRecord.ok !== false; if (preflight) { - if (passed) this.approved.set(plan.artifactKey, plan); - else this.approved.delete(plan.artifactKey); + const approvedOperations = resultRecord?.approvedOperations; + const approvedPlan = Array.isArray(approvedOperations) + ? this.planFromArgs({ artifactId, operations: approvedOperations }, "verify") + : undefined; + const authoritative = approvedPlan + && approvedPlan.operations.length > 0 + && approvedPlan.operations.every((operation) => operation.baseVersion !== undefined) + && operationsMatchApprovedPlan(requestedPlan.operations, approvedPlan.operations, true); + if (passed && authoritative) this.approved.set(approvedPlan.artifactKey, approvedPlan); + else this.approved.delete(requestedPlan.artifactKey); return; } if (!passed) return; - const pending = this.pending.get(plan.artifactKey); - if (pending?.signature === plan.signature) { - this.pending.delete(plan.artifactKey); - this.verified.set(plan.artifactKey, pending); + const pending = this.pending.get(requestedPlan.artifactKey); + if (pending?.signature === requestedPlan.signature) { + this.pending.delete(requestedPlan.artifactKey); + this.verified.set(requestedPlan.artifactKey, pending); this.verifiedWrites += 1; - } else if (this.verified.get(plan.artifactKey)?.signature === plan.signature) { - this.verified.set(plan.artifactKey, plan); + } else if (this.verified.get(requestedPlan.artifactKey)?.signature === requestedPlan.signature) { + this.verified.set(requestedPlan.artifactKey, requestedPlan); } return; } @@ -320,14 +357,14 @@ class WorkbookWorkflowState { : Array.isArray(args.ops) ? args.ops : Array.isArray(args.cells) ? args.cells : [args]; - const operations = rawOperations - .flatMap((operation) => normalizeOperation(operation, artifactId)) + const operations = rawOperations.flatMap((operation) => normalizeOperation(operation, artifactId)); + const canonicalOperations = [...operations] .sort((left, right) => left.target.localeCompare(right.target) || stableStringify(left).localeCompare(stableStringify(right))); return { artifactId, artifactKey: artifactKey(artifactId), operations, - signature: stableStringify(operations), + signature: stableStringify(canonicalOperations), }; } @@ -355,10 +392,12 @@ function normalizeOperation(value: unknown, artifactId: string): WorkbookOperati : nested && formula ? undefined : ownValue(operation, ["value", "newValue", "new_value", "text", "content"]); const numFmt = stringField(operation, ["numFmt", "num_fmt", "numberFormat", "number_format"]) ?? (nested ? stringField(nested, ["numFmt", "num_fmt", "numberFormat", "number_format"]) : undefined); + const baseVersion = integerValue(ownValue(operation, ["baseVersion", "base_version", "currentVersion", "current_version", "version"])); const normalizedElementId = normalizeElementId(elementId); return [{ elementId: normalizedElementId, target: targetFor(normalizedElementId, artifactId), + ...(baseVersion !== undefined ? { baseVersion } : {}), ...(formula ? { formula } : {}), ...(formula && result !== undefined ? { result: normalizeJson(result) } : {}), ...(!formula && scalarValue !== undefined ? { value: normalizeJson(scalarValue) } : {}), @@ -393,11 +432,82 @@ function targetFor(elementId: string, artifactId: string): string { return elementId.includes("!") ? elementId : `${artifactKey(artifactId)}!${elementId}`; } -function writeContract(plan: WorkbookPlan): Record { - const ops = plan.operations.map(operationArgs); - return ops.length === 1 - ? { tool: "write_locked_cell", args: { artifactId: plan.artifactId, ...ops[0] } } - : { tool: "write_locked_cells", args: { artifactId: plan.artifactId, ops } }; +function bindApprovedBatchWrite(call: ToolCall, plan: WorkbookPlan): ToolCall { + return { + ...call, + tool: "write_locked_cells", + args: { artifactId: plan.artifactId, ops: plan.operations.map(operationArgs) }, + }; +} + +function bindPendingVerification(call: ToolCall, plan: WorkbookPlan): ToolCall { + const args = argsRecord(call.args) ?? {}; + return { + ...call, + args: { + ...args, + artifactId: plan.artifactId, + operations: plan.operations.map(operationArgs), + afterWrite: true, + }, + }; +} + +function artifactAllowsApprovedBinding(argsValue: unknown, approvedArtifactId: string): boolean { + const raw = argsRecord(argsValue)?.artifactId; + if (raw === undefined || raw === null || (typeof raw === "string" && !raw.trim())) return true; + if (typeof raw !== "string") return false; + if (raw.trim() === approvedArtifactId) return true; + if (!raw.startsWith(approvedArtifactId)) return false; + const suffix = raw.slice(approvedArtifactId.length); + return /^\s*(?:\r?\n|<(?:tool_call|arg_key|function|tool-use)\b)/i.test(suffix); +} + +function rawBatchOperationsMatchApprovedSubset(argsValue: unknown, approved: WorkbookPlan): boolean { + const args = argsRecord(argsValue); + if (!args || !Array.isArray(args.ops) || args.ops.length === 0 || args.ops.length > approved.operations.length) return false; + const requested: WorkbookOperation[] = []; + for (const rawOperation of args.ops) { + const normalized = normalizeOperation(rawOperation, approved.artifactId); + if (normalized.length !== 1) return false; + requested.push(normalized[0]); + } + + return operationsMatchApprovedPlan(requested, approved.operations, false); +} + +function operationsMatchApprovedPlan( + requested: WorkbookOperation[], + approved: WorkbookOperation[], + requireComplete: boolean, +): boolean { + if (requested.length === 0 || requested.length > approved.length) return false; + if (requireComplete && requested.length !== approved.length) return false; + const remaining = [...approved]; + for (const operation of requested) { + const matchIndex = remaining.findIndex((candidate) => operationMatchesApproved(operation, candidate)); + if (matchIndex === -1) return false; + remaining.splice(matchIndex, 1); + } + return !requireComplete || remaining.length === 0; +} + +function operationMatchesApproved(requested: WorkbookOperation, approved: WorkbookOperation): boolean { + if (requested.baseVersion !== undefined && requested.baseVersion !== approved.baseVersion) return false; + return stableStringify(operationArgsWithoutVersion(requested)) === stableStringify(operationArgsWithoutVersion(approved)); +} + +function operationArgsWithoutVersion(operation: WorkbookOperation): Record { + const args = operationArgs(operation); + delete args.baseVersion; + return args; +} + +function targetSummary(operations: WorkbookOperation[], limit = 8): { targets: string[]; omitted: number } { + return { + targets: operations.slice(0, limit).map((operation) => operation.target), + omitted: Math.max(0, operations.length - limit), + }; } function verificationArgs(plan: WorkbookPlan, instruction?: string): Record { @@ -412,6 +522,7 @@ function verificationArgs(plan: WorkbookPlan, instruction?: string): Record { return { elementId: operation.elementId, + ...(operation.baseVersion !== undefined ? { baseVersion: operation.baseVersion } : {}), ...(operation.formula ? { formula: operation.formula } : {}), ...(Object.prototype.hasOwnProperty.call(operation, "value") ? { value: operation.value } : {}), ...(Object.prototype.hasOwnProperty.call(operation, "result") ? { result: operation.result } : {}), @@ -521,6 +632,12 @@ function ownValue(record: Record, keys: string[]): unknown { return undefined; } +function integerValue(value: unknown): number | undefined { + if (value === undefined || value === null || value === "") return undefined; + const parsed = Number(value); + return Number.isInteger(parsed) ? parsed : undefined; +} + function normalizeJson(value: unknown): unknown { if (Array.isArray(value)) return value.map(normalizeJson); const record = asRecord(value); diff --git a/src/nodeagent/models/convexModel.ts b/src/nodeagent/models/convexModel.ts index c75f9f38..125523b8 100644 --- a/src/nodeagent/models/convexModel.ts +++ b/src/nodeagent/models/convexModel.ts @@ -7,7 +7,7 @@ * file implements the small AgentModel seam with direct provider HTTP calls. */ -import type { AgentMessage, AgentModel, AgentStep, AgentTool, AgentToolChoice, ToolCall } from "../core/types"; +import type { AgentMessage, AgentModel, AgentModelRouteState, AgentStep, AgentTool, AgentToolChoice, TokenUsage, ToolCall } from "../core/types"; import { getModelPricing, getProviderForModel, resolveModelAlias } from "./modelCatalog"; import { isOpenRouterFreeAutoModel, @@ -55,6 +55,8 @@ type OpenAiChatResponse = { completion_tokens?: number; input_tokens?: number; output_tokens?: number; + prompt_tokens_details?: { cached_tokens?: number }; + input_tokens_details?: { cached_tokens?: number }; }; }; @@ -80,7 +82,12 @@ type AnthropicResponse = { | { type: "text"; text?: string } | { type: "tool_use"; id?: string; name?: string; input?: JsonObject } >; - usage?: { input_tokens?: number; output_tokens?: number }; + usage?: { + input_tokens?: number; + output_tokens?: number; + cache_read_input_tokens?: number; + cache_creation_input_tokens?: number; + }; }; type GeminiResponse = { @@ -95,6 +102,7 @@ type GeminiResponse = { usageMetadata?: { promptTokenCount?: number; candidatesTokenCount?: number; + cachedContentTokenCount?: number; totalTokenCount?: number; }; }; @@ -107,22 +115,78 @@ const TRANSIENT_RE = /(\b429\b|\b5\d\d\b|rate.?limit|overloaded|temporar|timed?. export type ConvexModelOptions = { entrypoint?: ProviderRouteEntrypoint; freeAutoMode?: OpenRouterFreeModelMode; + /** Ordered, already-authorized fallback routes. Pass [] to keep an explicit model exact. */ + fallbackModelIds?: string[]; + /** Checkpointed concrete-route preference/cooldowns from a prior durable slice. */ + routeState?: AgentModelRouteState; }; +type ConcreteRouteState = { + preferredModelId: string; + cooldownUntil: Map; +}; + +class ProviderStepError extends Error { + readonly cause: unknown; + + constructor(cause: unknown, readonly usage?: TokenUsage) { + super(cause instanceof Error ? cause.message : String(cause)); + this.name = "ProviderStepError"; + this.cause = cause; + } +} + +function openAiUsage(usage?: OpenAiChatResponse["usage"]): TokenUsage { + return { + inputTokens: usage?.prompt_tokens ?? usage?.input_tokens ?? 0, + outputTokens: usage?.completion_tokens ?? usage?.output_tokens ?? 0, + cachedInputTokens: usage?.prompt_tokens_details?.cached_tokens + ?? usage?.input_tokens_details?.cached_tokens + ?? 0, + }; +} + +function anthropicUsage(usage?: AnthropicResponse["usage"]): TokenUsage { + const uncached = usage?.input_tokens ?? 0; + const cached = usage?.cache_read_input_tokens ?? 0; + const cacheCreation = usage?.cache_creation_input_tokens ?? 0; + return { + inputTokens: uncached + cached + cacheCreation, + outputTokens: usage?.output_tokens ?? 0, + cachedInputTokens: cached, + cacheCreationInputTokens: cacheCreation, + }; +} + +function geminiUsage(usage?: GeminiResponse["usageMetadata"]): TokenUsage { + return { + inputTokens: usage?.promptTokenCount ?? 0, + outputTokens: usage?.candidatesTokenCount ?? 0, + cachedInputTokens: usage?.cachedContentTokenCount ?? 0, + }; +} + export function convexModel(modelId: string, options: ConvexModelOptions = {}): AgentModel { const aliasModelId = resolveModelAlias(modelId); const entrypoint = options.entrypoint ?? "system"; const freeAutoMode = options.freeAutoMode; + // Fallbacks must be authorized by the caller that owns artifact-aware egress context. + // Non-durable/direct call sites remain exact even when deployment fallback env vars exist. + const fallbackModelIds = normalizeFallbackModelIds(aliasModelId, options.fallbackModelIds ?? []); + const concreteRouteState = hydrateConcreteRouteState(aliasModelId, fallbackModelIds, options.routeState); let resolvedModelId = aliasModelId; return { get name() { return resolvedModelId; }, + routeState() { + return snapshotConcreteRouteState(concreteRouteState); + }, async next({ system, messages, tools, signal, onTextDelta, toolChoice }) { // Gateway PII firewall — redact PII/secrets from the system + user content before the prompt leaves. const safeSystem = redactPII(system).text; const safeMessages = messages.map((m) => (m.role === "user" && m.content ? { ...m, content: redactPII(m.content).text } : m)); - const { step, resolvedModel } = await generateConvexAgentStep(aliasModelId, safeSystem, safeMessages, tools, entrypoint, signal, onTextDelta, toolChoice, freeAutoMode); + const { step, resolvedModel } = await generateConvexAgentStep(aliasModelId, safeSystem, safeMessages, tools, entrypoint, signal, onTextDelta, toolChoice, freeAutoMode, fallbackModelIds, concreteRouteState); resolvedModelId = resolvedModel; return step; }, @@ -134,6 +198,24 @@ export function convexPriceRun(modelId: string, inTok: number, outTok: number): return (inTok * (pricing?.inputPer1M ?? 1) + outTok * (pricing?.outputPer1M ?? 5)) / 1_000_000; } +function exactConvexUsageCost( + modelId: string, + usage: Pick, +): number | undefined { + const pricing = getModelPricing(resolveModelAlias(modelId)); + if (!pricing) return undefined; + // The catalog does not yet carry provider cache-write rates. Never label a + // cache-creation turn exact by silently charging it at the read/input rate. + if ((usage.cacheCreationInputTokens ?? 0) > 0) return undefined; + const cachedInputTokens = Math.min(usage.inputTokens, Math.max(0, usage.cachedInputTokens ?? 0)); + const uncachedInputTokens = Math.max(0, usage.inputTokens - cachedInputTokens); + return ( + uncachedInputTokens * pricing.inputPer1M + + cachedInputTokens * (pricing.cachedInputPer1M ?? pricing.inputPer1M) + + usage.outputTokens * pricing.outputPer1M + ) / 1_000_000; +} + async function generateConvexAgentStep( modelId: string, system: string, @@ -144,11 +226,23 @@ async function generateConvexAgentStep( onTextDelta?: (text: string) => void | Promise, toolChoice?: AgentToolChoice, freeAutoMode?: OpenRouterFreeModelMode, + fallbackModelIds: string[] = [], + concreteRouteState?: ConcreteRouteState, ) { assertProviderRouteAllowed({ model: modelId, entrypoint, env: process.env }); + let providerRequests = 0; + const onProviderRequest = () => { + providerRequests += 1; + }; if (isOpenRouterFreeAutoModel(modelId)) { const requestStartedAt = Date.now(); const requestSignal = openRouterFreeRequestSignal(signal); + const candidateText = acceptedCandidateTextBuffer(onTextDelta); + let aggregateInputTokens = 0; + let aggregateOutputTokens = 0; + let aggregateCachedInputTokens = 0; + let aggregateCacheCreationInputTokens = 0; + let aggregateCostKnown = true; const candidates = await selectOpenRouterFreeModels({ mode: freeAutoMode ?? (tools.length ? "agent" : "chat"), limit: openRouterFreeAutoLimit(), @@ -167,7 +261,10 @@ async function generateConvexAgentStep( const providerRoute = assertProviderRouteAllowed({ model: candidate.id, entrypoint, env: process.env }); // The controller owns this deadline; do not wrap it with openRouterFreeCandidateSignal. const candidateSignal = context.signal; - return withProviderRoute(await withRetry(() => openAiCompatibleStep({ + const requestsBefore = providerRequests; + let step: AgentStep; + try { + step = withProviderRoute(await withRetry(() => openAiCompatibleStep({ endpoint: `${openRouterBaseUrl()}/chat/completions`, apiKey: envValue("OPENROUTER_API_KEY"), headers: openRouterHeaders(), @@ -176,22 +273,36 @@ async function generateConvexAgentStep( messages, tools, signal: candidateSignal, - onTextDelta, + onTextDelta: candidateText.sink(candidate.id), toolChoice, + onProviderRequest, }), candidateSignal, 0), providerRoute); + } catch (error) { + const usage = errorTokenUsage(error); + accumulateTokenUsage(usage, { + input: (value) => { aggregateInputTokens += value; }, + output: (value) => { aggregateOutputTokens += value; }, + cached: (value) => { aggregateCachedInputTokens += value; }, + cacheCreation: (value) => { aggregateCacheCreationInputTokens += value; }, + }); + if (providerRequests > requestsBefore && !usage) aggregateCostKnown = false; + throw error; + } + accumulateTokenUsage(step.usage, { + input: (value) => { aggregateInputTokens += value; }, + output: (value) => { aggregateOutputTokens += value; }, + cached: (value) => { aggregateCachedInputTokens += value; }, + cacheCreation: (value) => { aggregateCacheCreationInputTokens += value; }, + }); + return step; }, - assessResult: (step) => assessAgentToolTurnQuality({ - text: step.text, - toolCalls: step.toolCalls, - tools, - messages, - requiredToolCall: toolChoice === "required", - }), + assessResult: (step) => assessConvexTurnQuality(step, tools, messages, toolChoice), classifyProviderFailure: (error) => { const failure = classifyQualityFailoverProviderError(error); return isProviderNonRetryableError(error) ? { ...failure, scope: "global" } : failure; }, - onRouteAttempt: (attempt, candidate) => { + onRouteAttempt: async (attempt, candidate) => { + await candidateText.settle(candidate.id, attempt.outcome === "accepted"); if ( attempt.outcome !== "accepted" && attempt.outcome !== "provider_failure" @@ -206,55 +317,176 @@ async function generateConvexAgentStep( }); }, }); + if (providerRequests > 0 && failover.receipt.routeAttempts.some((attempt) => attempt.reason === "candidate_timeout")) { + aggregateCostKnown = false; + } if (failover.ok) { return { step: { ...failover.result, + usage: { + inputTokens: aggregateInputTokens, + outputTokens: aggregateOutputTokens, + cachedInputTokens: aggregateCachedInputTokens, + cacheCreationInputTokens: aggregateCacheCreationInputTokens, + modelCalls: providerRequests, + ...(aggregateCostKnown + ? { costUsd: failover.receipt.budget.spentCostUsd, costKind: "exact" as const } + : { costUsd: failover.receipt.budget.spentCostUsd, costKind: "estimated" as const }), + }, providerRoute: { - ...failover.result.providerRoute, + ...(failover.result.providerRoute as ProviderRouteReceipt), qualityFailover: failover.receipt, }, }, resolvedModel: failover.candidate.id, }; } - throw convexQualityFailoverError(failover.receipt, failover.lastError); + throw convexQualityFailoverError(failover.receipt, failover.lastError, "openrouter/free-auto", { + inputTokens: aggregateInputTokens, + outputTokens: aggregateOutputTokens, + cachedInputTokens: aggregateCachedInputTokens, + cacheCreationInputTokens: aggregateCacheCreationInputTokens, + modelCalls: providerRequests, + costUsd: failover.receipt.budget.spentCostUsd, + costKind: aggregateCostKnown ? "exact" as const : "estimated" as const, + }); } - try { - const providerRoute = assertProviderRouteAllowed({ model: modelId, entrypoint, env: process.env }); - return { - step: withProviderRoute(await withRetry(() => providerStep(modelId, system, messages, tools, signal, onTextDelta, toolChoice), signal), providerRoute), - resolvedModel: modelId, - }; - } catch (error) { - const fb = fallbackModelFor(modelId); - if (!fb || signal?.aborted) throw error; - const providerRoute = assertProviderRouteAllowed({ model: fb, entrypoint, env: process.env }); - return { - step: withProviderRoute(await withRetry(() => providerStep(fb, system, messages, tools, signal, onTextDelta, toolChoice), signal), providerRoute), - resolvedModel: fb, - }; + const state = concreteRouteState ?? { preferredModelId: modelId, cooldownUntil: new Map() }; + const candidateIds = orderedConcreteCandidateIds(modelId, fallbackModelIds, state.preferredModelId); + const candidates = candidateIds.map((id) => ({ + id, + provider: getProviderForModel(id) ?? undefined, + cooldownUntil: state.cooldownUntil.get(id), + })); + let aggregateInputTokens = 0; + let aggregateOutputTokens = 0; + let aggregateCachedInputTokens = 0; + let aggregateCacheCreationInputTokens = 0; + let aggregateCostKnown = true; + const candidateText = acceptedCandidateTextBuffer(onTextDelta); + const failover = await runQualityFailover({ + candidates, + budget: { maxAttempts: candidateIds.length }, + attemptTimeoutMs: concreteCandidateTimeoutMs(), + signal, + execute: async (candidate, context) => { + const providerRoute = assertProviderRouteAllowed({ model: candidate.id, entrypoint, env: process.env }); + const requestsBefore = providerRequests; + let step: AgentStep; + try { + step = withProviderRoute( + await withRetry( + () => providerStep(candidate.id, system, messages, tools, context.signal, candidateText.sink(candidate.id), toolChoice, onProviderRequest), + context.signal, + 0, + ), + providerRoute, + ); + } catch (error) { + const usage = errorTokenUsage(error); + accumulateTokenUsage(usage, { + input: (value) => { aggregateInputTokens += value; }, + output: (value) => { aggregateOutputTokens += value; }, + cached: (value) => { aggregateCachedInputTokens += value; }, + cacheCreation: (value) => { aggregateCacheCreationInputTokens += value; }, + }); + if (providerRequests > requestsBefore && !usage) aggregateCostKnown = false; + throw error; + } + accumulateTokenUsage(step.usage, { + input: (value) => { aggregateInputTokens += value; }, + output: (value) => { aggregateOutputTokens += value; }, + cached: (value) => { aggregateCachedInputTokens += value; }, + cacheCreation: (value) => { aggregateCacheCreationInputTokens += value; }, + }); + return step; + }, + assessResult: (step) => assessConvexTurnQuality(step, tools, messages, toolChoice), + classifyProviderFailure: (error, candidate) => { + const failure = classifyQualityFailoverProviderError(error); + // Auth/quota are provider-account local when an explicitly authorized cross-provider + // candidate exists. Policy/egress failures are global and must never be routed around. + const candidateIndex = candidateIds.indexOf(candidate.id); + const hasCrossProviderCandidate = candidateIndex >= 0 && candidateIds.slice(candidateIndex + 1).some((id) => { + const provider = getProviderForModel(id); + return !!provider && !!candidate.provider && provider !== candidate.provider; + }); + return failure.scope === "global" + && (failure.category === "auth" || failure.category === "quota") + && hasCrossProviderCandidate + ? { ...failure, scope: "candidate" as const } + : failure; + }, + onRouteAttempt: async (attempt, candidate) => { + await candidateText.settle(candidate.id, attempt.outcome === "accepted"); + updateConcreteRouteState(state, candidate.id, attempt.outcome, attempt.providerFailureCategory); + }, + measureCostUsd: ({ candidate, result, error }) => { + const usage = result?.usage ?? errorTokenUsage(error); + if (!usage) return 0; + const measured = exactConvexUsageCost(candidate.id, usage); + if (measured === undefined) aggregateCostKnown = false; + return measured ?? 0; + }, + }); + if (providerRequests > 0 && failover.receipt.routeAttempts.some((attempt) => attempt.reason === "candidate_timeout")) { + aggregateCostKnown = false; + } + if (!failover.ok) { + throw convexQualityFailoverError(failover.receipt, failover.lastError ?? failover.lastResult, modelId, { + inputTokens: aggregateInputTokens, + outputTokens: aggregateOutputTokens, + cachedInputTokens: aggregateCachedInputTokens, + cacheCreationInputTokens: aggregateCacheCreationInputTokens, + modelCalls: providerRequests, + ...(aggregateCostKnown + ? { costUsd: failover.receipt.budget.spentCostUsd, costKind: "exact" as const } + : { costUsd: failover.receipt.budget.spentCostUsd, costKind: "estimated" as const }), + }); } + return { + step: { + ...failover.result, + usage: { + inputTokens: aggregateInputTokens, + outputTokens: aggregateOutputTokens, + cachedInputTokens: aggregateCachedInputTokens, + cacheCreationInputTokens: aggregateCacheCreationInputTokens, + modelCalls: providerRequests, + ...(aggregateCostKnown ? { costUsd: failover.receipt.budget.spentCostUsd, costKind: "exact" as const } : {}), + }, + providerRoute: { + ...(failover.result.providerRoute as ProviderRouteReceipt), + qualityFailover: failover.receipt, + }, + }, + resolvedModel: failover.candidate.id, + }; } function convexQualityFailoverError( receipt: QualityFailoverReceipt, cause?: unknown, + routeLabel = "openrouter/free-auto", + usage?: TokenUsage, ): QualityFailoverError { const attempted = receipt.routeAttempts.map((attempt) => attempt.routeId).join(", ") || "no route attempts"; if (receipt.stopReason === "time_budget") { return new QualityFailoverError( - `openrouter/free-auto request deadline exceeded after ${attempted}`, + `${routeLabel} request deadline exceeded after ${attempted}`, receipt, cause, + usage, ); } if (receipt.stopReason === "aborted") { return new QualityFailoverError( - `openrouter/free-auto request aborted after ${attempted}`, + `${routeLabel} request aborted after ${attempted}`, receipt, cause, + usage, ); } const failure = cause @@ -262,12 +494,123 @@ function convexQualityFailoverError( ?? receipt.terminalFailure?.reason ?? receipt.stopReason; return new QualityFailoverError( - `openrouter/free-auto failed for ${attempted}: ${shortProviderError(failure)}`, + `${routeLabel} failed for ${attempted}: ${shortProviderError(failure)}`, receipt, cause, + usage, ); } +function assessConvexTurnQuality( + step: AgentStep, + tools: AgentTool[], + messages: AgentMessage[], + toolChoice?: AgentToolChoice, +) { + // Required-tool protocol recovery belongs to runAgent, which can add corrective context, + // validate schemas as ordinary tool results, and terminate with a typed protocol_stall receipt. + // Rejecting here would turn a recoverable four-turn protocol into a generic one-call route error. + if (toolChoice === "required") return { ok: true as const }; + return assessAgentToolTurnQuality({ + text: step.text, + toolCalls: step.toolCalls, + tools, + messages, + requiredToolCall: false, + }); +} + +function orderedConcreteCandidateIds( + primaryModelId: string, + fallbackModelIds: readonly string[], + preferredModelId: string, +): string[] { + const authorized = [primaryModelId, ...normalizeFallbackModelIds(primaryModelId, fallbackModelIds)]; + if (!authorized.includes(preferredModelId)) return authorized; + return [preferredModelId, ...authorized.filter((id) => id !== preferredModelId)]; +} + +function hydrateConcreteRouteState( + primaryModelId: string, + fallbackModelIds: readonly string[], + persisted?: AgentModelRouteState, +): ConcreteRouteState { + const allowed = [primaryModelId, ...normalizeFallbackModelIds(primaryModelId, fallbackModelIds)]; + const preferred = persisted?.preferredModelId + ? resolveModelAlias(persisted.preferredModelId) + : primaryModelId; + const now = Date.now(); + const cooldownUntil = new Map(); + for (const [candidateId, until] of Object.entries(persisted?.cooldownUntil ?? {})) { + const normalized = resolveModelAlias(candidateId); + if (allowed.includes(normalized) && Number.isFinite(until) && until > now) { + cooldownUntil.set(normalized, until); + } + } + return { + preferredModelId: allowed.includes(preferred) ? preferred : primaryModelId, + cooldownUntil, + }; +} + +function snapshotConcreteRouteState(state: ConcreteRouteState): AgentModelRouteState { + const now = Date.now(); + const cooldownUntil: Record = {}; + for (const [candidateId, until] of state.cooldownUntil) { + if (Number.isFinite(until) && until > now) cooldownUntil[candidateId] = until; + } + return { + preferredModelId: state.preferredModelId, + ...(Object.keys(cooldownUntil).length ? { cooldownUntil } : {}), + }; +} + +function errorTokenUsage(error: unknown): TokenUsage | undefined { + if (error instanceof ProviderStepError) return error.usage; + if (!error || typeof error !== "object") return undefined; + const usage = (error as { usage?: unknown }).usage; + if (!usage || typeof usage !== "object") return undefined; + const candidate = usage as Partial; + if (!Number.isFinite(candidate.inputTokens) || !Number.isFinite(candidate.outputTokens)) return undefined; + return candidate as TokenUsage; +} + +function accumulateTokenUsage( + usage: TokenUsage | undefined, + sinks: { + input(value: number): void; + output(value: number): void; + cached(value: number): void; + cacheCreation(value: number): void; + }, +): void { + if (!usage) return; + sinks.input(usage.inputTokens ?? 0); + sinks.output(usage.outputTokens ?? 0); + sinks.cached(usage.cachedInputTokens ?? 0); + sinks.cacheCreation(usage.cacheCreationInputTokens ?? 0); +} + +function updateConcreteRouteState( + state: ConcreteRouteState, + candidateId: string, + outcome: "accepted" | "provider_failure" | "quality_failure" | "control_failure" | "aborted", + providerFailureCategory?: string, +): void { + if (outcome === "accepted") { + state.preferredModelId = candidateId; + state.cooldownUntil.delete(candidateId); + return; + } + if (outcome !== "provider_failure" && outcome !== "quality_failure") return; + const cooldownMs = providerFailureCategory === "auth" || providerFailureCategory === "quota" + ? 5 * 60_000 + : outcome === "quality_failure" + ? 60_000 + : 30_000; + state.cooldownUntil.set(candidateId, Date.now() + cooldownMs); +} + async function providerStep( modelId: string, system: string, @@ -276,6 +619,7 @@ async function providerStep( signal?: AbortSignal, onTextDelta?: (text: string) => void | Promise, toolChoice?: AgentToolChoice, + onProviderRequest?: () => void, ) { const provider = getProviderForModel(modelId); if (provider === "openai") { @@ -290,6 +634,7 @@ async function providerStep( signal, onTextDelta, toolChoice, + onProviderRequest, }); } if (provider === "openrouter") { @@ -304,6 +649,7 @@ async function providerStep( signal, onTextDelta, toolChoice, + onProviderRequest, }); } if (provider === "nebius") { @@ -319,18 +665,20 @@ async function providerStep( signal, onTextDelta, toolChoice, + onProviderRequest, }); } - if (provider === "anthropic") return anthropicStep(modelId, system, messages, tools, signal); + if (provider === "anthropic") return anthropicStep(modelId, system, messages, tools, signal, onProviderRequest); if (provider === "gemini") { if (onTextDelta) { - try { - return await geminiStreamStep(modelId, system, messages, tools, signal, onTextDelta); - } catch (error) { - if (signal?.aborted) throw error; - } + const deltas: string[] = []; + const step = await geminiStreamStep(modelId, system, messages, tools, signal, (text) => { + deltas.push(text); + }, onProviderRequest); + await emitBufferedText(deltas, onTextDelta); + return step; } - return geminiStep(modelId, system, messages, tools, signal); + return geminiStep(modelId, system, messages, tools, signal, onProviderRequest); } throw new Error(`convexModel(): no provider for "${modelId}"`); } @@ -346,20 +694,52 @@ async function openAiCompatibleStep(args: { signal?: AbortSignal; onTextDelta?: (text: string) => void | Promise; toolChoice?: AgentToolChoice; + onProviderRequest?: () => void; }) { if (args.onTextDelta) { - try { - return await openAiCompatibleStreamStep({ ...args, onTextDelta: args.onTextDelta }); - } catch (error) { - if (args.signal?.aborted) throw error; - // Some OpenAI-compatible providers/models reject stream_options or tool streaming. Keep the - // durable job reliable by falling back to the established blocking request path. - } + const deltas: string[] = []; + const step = await openAiCompatibleStreamStep({ ...args, onTextDelta: (text) => { + deltas.push(text); + } }); + await emitBufferedText(deltas, args.onTextDelta); + return step; } return openAiCompatibleBlockingStep(args); } +function acceptedCandidateTextBuffer(onTextDelta?: (text: string) => void | Promise) { + const chunks = new Map(); + return { + sink(candidateId: string): ((text: string) => void) | undefined { + if (!onTextDelta) return undefined; + const buffered: string[] = []; + chunks.set(candidateId, buffered); + return (text: string) => { + if (text) buffered.push(text); + }; + }, + async settle(candidateId: string, accepted: boolean): Promise { + const buffered = chunks.get(candidateId) ?? []; + chunks.delete(candidateId); + if (accepted && onTextDelta) await emitBufferedText(buffered, onTextDelta); + }, + }; +} + +async function emitBufferedText( + chunks: readonly string[], + onTextDelta: (text: string) => void | Promise, +): Promise { + for (const chunk of chunks) { + try { + await onTextDelta(chunk); + } catch { + // Public stream telemetry must not change model routing or tool execution. + } + } +} + async function openAiCompatibleBlockingStep(args: { endpoint: string; apiKey?: string; @@ -370,6 +750,7 @@ async function openAiCompatibleBlockingStep(args: { tools: AgentTool[]; signal?: AbortSignal; toolChoice?: AgentToolChoice; + onProviderRequest?: () => void; }) { const res = await postJson(args.endpoint, { model: args.modelId, @@ -381,7 +762,7 @@ async function openAiCompatibleBlockingStep(args: { }, { ...args.headers, ...(args.apiKey ? { Authorization: `Bearer ${args.apiKey}` } : {}), - }, args.signal); + }, args.signal, args.onProviderRequest); const message = res.choices?.[0]?.message ?? {}; const toolCalls = (message.tool_calls ?? []).map((tc): ToolCall => ({ @@ -396,6 +777,9 @@ async function openAiCompatibleBlockingStep(args: { usage: { inputTokens: res.usage?.prompt_tokens ?? res.usage?.input_tokens ?? 0, outputTokens: res.usage?.completion_tokens ?? res.usage?.output_tokens ?? 0, + cachedInputTokens: res.usage?.prompt_tokens_details?.cached_tokens + ?? res.usage?.input_tokens_details?.cached_tokens + ?? 0, }, }; } @@ -411,7 +795,9 @@ async function openAiCompatibleStreamStep(args: { signal?: AbortSignal; onTextDelta: (text: string) => void | Promise; toolChoice?: AgentToolChoice; + onProviderRequest?: () => void; }) { + args.onProviderRequest?.(); const res = await fetch(args.endpoint, { method: "POST", headers: { @@ -437,32 +823,36 @@ async function openAiCompatibleStreamStep(args: { let text = ""; let usage: OpenAiChatResponse["usage"] | undefined; - await readSse(res, async (data) => { - let parsed: OpenAiChatStreamChunk; - try { - parsed = JSON.parse(data) as OpenAiChatStreamChunk; - } catch { - return; - } - if (parsed.usage) usage = parsed.usage; - for (const choice of parsed.choices ?? []) { - const delta = choice.delta; - const textDelta = delta?.content ?? ""; - if (textDelta) { - text += textDelta; - await args.onTextDelta(textDelta); + try { + await readSse(res, async (data) => { + let parsed: OpenAiChatStreamChunk; + try { + parsed = JSON.parse(data) as OpenAiChatStreamChunk; + } catch { + return; } - for (const toolDelta of delta?.tool_calls ?? []) { - const index = inferOpenAiStreamToolIndex(toolDelta, toolCallParts, lastToolCallIndex); - lastToolCallIndex = index; - const current = toolCallParts.get(index) ?? { argsText: "" }; - if (toolDelta.id) current.id = toolDelta.id; - if (toolDelta.function?.name) current.name = toolDelta.function.name; - if (toolDelta.function?.arguments) current.argsText += toolDelta.function.arguments; - toolCallParts.set(index, current); + if (parsed.usage) usage = parsed.usage; + for (const choice of parsed.choices ?? []) { + const delta = choice.delta; + const textDelta = delta?.content ?? ""; + if (textDelta) { + text += textDelta; + await args.onTextDelta(textDelta); + } + for (const toolDelta of delta?.tool_calls ?? []) { + const index = inferOpenAiStreamToolIndex(toolDelta, toolCallParts, lastToolCallIndex); + lastToolCallIndex = index; + const current = toolCallParts.get(index) ?? { argsText: "" }; + if (toolDelta.id) current.id = toolDelta.id; + if (toolDelta.function?.name) current.name = toolDelta.function.name; + if (toolDelta.function?.arguments) current.argsText += toolDelta.function.arguments; + toolCallParts.set(index, current); + } } - } - }); + }); + } catch (error) { + throw new ProviderStepError(error, usage ? openAiUsage(usage) : undefined); + } const toolCalls = [...toolCallParts.entries()] .sort(([a], [b]) => a - b) @@ -476,10 +866,7 @@ async function openAiCompatibleStreamStep(args: { text: text || undefined, toolCalls, done: toolCalls.length === 0, - usage: { - inputTokens: usage?.prompt_tokens ?? usage?.input_tokens ?? 0, - outputTokens: usage?.completion_tokens ?? usage?.output_tokens ?? 0, - }, + usage: openAiUsage(usage), }; } @@ -489,6 +876,7 @@ async function anthropicStep( messages: AgentMessage[], tools: AgentTool[], signal?: AbortSignal, + onProviderRequest?: () => void, ) { const res = await postJson("https://api.anthropic.com/v1/messages", { model: modelId, @@ -499,7 +887,7 @@ async function anthropicStep( }, { "x-api-key": requireEnv("ANTHROPIC_API_KEY"), "anthropic-version": "2023-06-01", - }, signal); + }, signal, onProviderRequest); const parts = res.content ?? []; const text = parts @@ -517,10 +905,7 @@ async function anthropicStep( text: text || undefined, toolCalls, done: toolCalls.length === 0, - usage: { - inputTokens: res.usage?.input_tokens ?? 0, - outputTokens: res.usage?.output_tokens ?? 0, - }, + usage: anthropicUsage(res.usage), }; } @@ -530,6 +915,7 @@ async function geminiStep( messages: AgentMessage[], tools: AgentTool[], signal?: AbortSignal, + onProviderRequest?: () => void, ) { const url = `https://generativelanguage.googleapis.com/v1beta/models/${encodeURIComponent(modelId)}:generateContent?key=${encodeURIComponent(requireEnv("GOOGLE_GENERATIVE_AI_API_KEY"))}`; const res = await postJson(url, { @@ -537,7 +923,7 @@ async function geminiStep( contents: toGeminiContents(messages), tools: tools.length ? [{ functionDeclarations: tools.map(geminiTool) }] : undefined, generationConfig: { maxOutputTokens: modelMaxOutputTokens() }, - }, {}, signal); + }, {}, signal, onProviderRequest); const parts = res.candidates?.[0]?.content?.parts ?? []; const text = parts @@ -559,6 +945,7 @@ async function geminiStep( usage: { inputTokens: res.usageMetadata?.promptTokenCount ?? 0, outputTokens: res.usageMetadata?.candidatesTokenCount ?? 0, + cachedInputTokens: res.usageMetadata?.cachedContentTokenCount ?? 0, }, }; } @@ -570,8 +957,10 @@ async function geminiStreamStep( tools: AgentTool[], signal: AbortSignal | undefined, onTextDelta: (text: string) => void | Promise, + onProviderRequest?: () => void, ) { const url = `https://generativelanguage.googleapis.com/v1beta/models/${encodeURIComponent(modelId)}:streamGenerateContent?alt=sse&key=${encodeURIComponent(requireEnv("GOOGLE_GENERATIVE_AI_API_KEY"))}`; + onProviderRequest?.(); const res = await fetch(url, { method: "POST", headers: { "Content-Type": "application/json" }, @@ -588,41 +977,42 @@ async function geminiStreamStep( const toolCalls: ToolCall[] = []; let usage: GeminiResponse["usageMetadata"] | undefined; - await readSse(res, async (data) => { - let parsed: GeminiResponse; - try { - parsed = JSON.parse(data) as GeminiResponse; - } catch { - return; - } - if (parsed.usageMetadata) usage = parsed.usageMetadata; - const parts = parsed.candidates?.[0]?.content?.parts ?? []; - for (const part of parts) { - if ("text" in part) { - const delta = part.text ?? ""; - if (delta) { - text += delta; - await onTextDelta(delta); + try { + await readSse(res, async (data) => { + let parsed: GeminiResponse; + try { + parsed = JSON.parse(data) as GeminiResponse; + } catch { + return; + } + if (parsed.usageMetadata) usage = parsed.usageMetadata; + const parts = parsed.candidates?.[0]?.content?.parts ?? []; + for (const part of parts) { + if ("text" in part) { + const delta = part.text ?? ""; + if (delta) { + text += delta; + await onTextDelta(delta); + } + } else if ("functionCall" in part) { + toolCalls.push({ + id: crypto.randomUUID(), + tool: part.functionCall?.name ?? "unknown_tool", + args: part.functionCall?.args ?? {}, + providerMetadata: part.thoughtSignature || part.thought_signature ? { geminiThoughtSignature: part.thoughtSignature ?? part.thought_signature } : undefined, + }); } - } else if ("functionCall" in part) { - toolCalls.push({ - id: crypto.randomUUID(), - tool: part.functionCall?.name ?? "unknown_tool", - args: part.functionCall?.args ?? {}, - providerMetadata: part.thoughtSignature || part.thought_signature ? { geminiThoughtSignature: part.thoughtSignature ?? part.thought_signature } : undefined, - }); } - } - }); + }); + } catch (error) { + throw new ProviderStepError(error, usage ? geminiUsage(usage) : undefined); + } return { text: text || undefined, toolCalls, done: toolCalls.length === 0, - usage: { - inputTokens: usage?.promptTokenCount ?? 0, - outputTokens: usage?.candidatesTokenCount ?? 0, - }, + usage: geminiUsage(usage), }; } @@ -922,7 +1312,7 @@ export function toolParameters(toolName: string): JsonObject { const stringOrStringArray = { anyOf: [stringArray, string] }; const workbookOperation = { type: "object", - properties: { elementId: string, value: any, formula: string, result: any, numFmt: string }, + properties: { elementId: string, baseVersion: integer, value: any, formula: string, result: any, numFmt: string }, required: ["elementId"], }; const schemas: Record = { @@ -1235,7 +1625,14 @@ async function readSse(res: Response, onData: (data: string) => Promise): if (buffer.trim()) await processLine(buffer); } -async function postJson(url: string, body: unknown, headers: Record, signal?: AbortSignal): Promise { +async function postJson( + url: string, + body: unknown, + headers: Record, + signal?: AbortSignal, + onProviderRequest?: () => void, +): Promise { + onProviderRequest?.(); const res = await fetch(url, { method: "POST", headers: { @@ -1370,9 +1767,32 @@ function openRouterHeaders(): Record { }; } -function fallbackModelFor(modelId: string): string | undefined { - const fb = process.env.AGENT_FALLBACK_MODEL?.trim(); - return fb && resolveModelAlias(fb) !== modelId ? resolveModelAlias(fb) : undefined; +export function configuredConvexModelFallbacks( + modelId: string, + env: NodeJS.ProcessEnv = process.env, +): string[] { + const raw = env.AGENT_FALLBACK_MODELS?.trim() || env.AGENT_FALLBACK_MODEL?.trim() || ""; + return normalizeFallbackModelIds(modelId, raw.split(/[\r\n,]+/)); +} + +function normalizeFallbackModelIds(modelId: string, values: readonly string[]): string[] { + const primary = resolveModelAlias(modelId); + const seen = new Set([primary]); + const result: string[] = []; + for (const value of values) { + if (typeof value !== "string" || !value.trim()) continue; + const resolved = resolveModelAlias(value.trim()); + if (seen.has(resolved)) continue; + seen.add(resolved); + result.push(resolved); + if (result.length >= 3) break; + } + return result; +} + +function concreteCandidateTimeoutMs(): number { + const raw = Number(envValue("AGENT_QUALITY_CANDIDATE_TIMEOUT_MS") ?? 120_000); + return Number.isFinite(raw) ? Math.max(10_000, Math.min(240_000, raw)) : 120_000; } function openRouterFreeAutoLimit(): number { diff --git a/src/nodeagent/models/phaseModel.ts b/src/nodeagent/models/phaseModel.ts index b2e2a850..73e1151d 100644 --- a/src/nodeagent/models/phaseModel.ts +++ b/src/nodeagent/models/phaseModel.ts @@ -24,4 +24,14 @@ export function modelForFramePhase( return fallback; } +export function authorizedModelForFramePhase( + phase: string, + fallback: string, + isAllowed: (model: string) => boolean, + env: Record = typeof process !== "undefined" ? process.env : {}, +): string { + const candidate = modelForFramePhase(phase, fallback, env); + return candidate === fallback || isAllowed(candidate) ? candidate : fallback; +} + export { ORCHESTRATOR_PHASES }; diff --git a/src/nodeagent/models/qualityFailover.ts b/src/nodeagent/models/qualityFailover.ts index 837202da..f05c952e 100644 --- a/src/nodeagent/models/qualityFailover.ts +++ b/src/nodeagent/models/qualityFailover.ts @@ -1,5 +1,5 @@ import { providerNonRetryableReason } from "../guardrails/egressPolicy"; -import type { AgentMessage, AgentTool, ToolCall } from "../core/types"; +import type { AgentMessage, AgentTool, TokenUsage, ToolCall } from "../core/types"; export const QUALITY_FAILOVER_SCHEMA = "nodeagent-quality-failover-v1" as const; export const QUALITY_FAILOVER_POLICY = "bounded_quality_failover_v1" as const; @@ -148,6 +148,8 @@ export class QualityFailoverError extends Error { message: string, readonly receipt: QualityFailoverReceipt, cause?: unknown, + /** Provider usage already incurred before routing stopped without an accepted result. */ + readonly usage?: TokenUsage, ) { super(message, cause === undefined ? undefined : { cause }); this.name = "QualityFailoverError"; @@ -220,6 +222,7 @@ type AttemptWork = | { kind: "assessment_error"; result: TResult; error: unknown }; type AttemptInterruptCause = "parent" | "time_budget" | "attempt_timeout"; +const INTERRUPTED_ATTEMPT_SETTLEMENT_GRACE_MS = 250; class AttemptInterruptedError extends Error { constructor(readonly cause: AttemptInterruptCause) { @@ -569,11 +572,22 @@ export async function runQualityFailover< scope.dispose(); if (!(error instanceof AttemptInterruptedError)) throw error; + // Aborting fetch/stream readers normally rejects the provider promise on + // the next turn. Give it a small bounded window so adapters can attach + // partial usage before we persist the timeout receipt. A provider that + // ignores AbortSignal still cannot hold the failover loop indefinitely. + const interruptedOutcome = await settleInterruptedAttempt(work); + const accountingError = interruptedOutcome?.kind === "provider_error" + ? interruptedOutcome.error + : interruptedOutcome?.kind === "assessment_error" + ? interruptedOutcome.error + : error; + const costUsd = measureCost(options, { candidate, attempt, estimatedCostUsd, - error, + error: accountingError, }); spentCostUsd = stableCost(spentCostUsd + costUsd); if (error.cause === "attempt_timeout") { @@ -599,7 +613,7 @@ export async function runQualityFailover< }); routeAttempts.push(attemptReceipt); await options.onRouteAttempt?.(attemptReceipt, candidate); - lastError = error; + lastError = accountingError; lastFailure = providerTerminalFailure(providerFailure); continue; } @@ -620,7 +634,7 @@ export async function runQualityFailover< }); routeAttempts.push(attemptReceipt); await options.onRouteAttempt?.(attemptReceipt, candidate); - lastError = error; + lastError = accountingError; return failed("blocked", stopReason, controlFailure(reason)); } scope.dispose(); @@ -767,6 +781,22 @@ export async function runQualityFailover< return failed("exhausted", "candidates_exhausted", lastFailure ?? controlFailure("candidates_exhausted")); } +async function settleInterruptedAttempt( + work: Promise>, +): Promise | undefined> { + let timer: ReturnType | undefined; + try { + return await Promise.race([ + work, + new Promise((resolve) => { + timer = setTimeout(() => resolve(undefined), INTERRUPTED_ATTEMPT_SETTLEMENT_GRACE_MS); + }), + ]); + } finally { + if (timer !== undefined) clearTimeout(timer); + } +} + function validateOptions( options: QualityFailoverOptions, ): { diff --git a/src/nodeagent/skills/spreadsheet/cellMutator.ts b/src/nodeagent/skills/spreadsheet/cellMutator.ts index ee6a7e30..1d57d535 100644 --- a/src/nodeagent/skills/spreadsheet/cellMutator.ts +++ b/src/nodeagent/skills/spreadsheet/cellMutator.ts @@ -866,6 +866,7 @@ const WRITE_LOCKED_CELL_RESULTS_TOOL: AgentTool = { const workbookOperationSchema = z.object({ elementId: z.string().min(1).describe("target cell id or A1 address"), + baseVersion: z.coerce.number().int().optional(), value: z.any().optional(), formula: z.string().nullish(), result: z.any().optional(), @@ -926,7 +927,7 @@ const VERIFY_WORKBOOK_TOOL: AgentTool = { execute: async (a: { instruction: string; artifactId?: string; - operations: Array<{ elementId: string; value?: unknown; formula?: string; result?: unknown; numFmt?: string }>; + operations: Array<{ elementId: string; baseVersion?: number; value?: unknown; formula?: string; result?: unknown; numFmt?: string }>; afterWrite?: boolean; }, rt) => { const snapshot = await rt.snapshot(a.artifactId); @@ -936,6 +937,14 @@ const VERIFY_WORKBOOK_TOOL: AgentTool = { const contextIds = searchHits.flatMap((hit) => hit.kind === "cell" ? [hit.elementId] : hit.elementIds); const confirmedIds = [...new Set([...elementIds, ...contextIds])]; const confirmed = await rt.readRange(confirmedIds, sheet); + const targetVersions = new Map(elementIds.map((elementId, index) => [elementId, confirmed[index]?.version])); + const targetsWithoutVersions = elementIds.filter((elementId) => !Number.isInteger(targetVersions.get(elementId))); + const targetsWithStaleVersions = a.operations.flatMap((operation) => { + const currentVersion = targetVersions.get(operation.elementId); + return operation.baseVersion !== undefined && currentVersion !== undefined && operation.baseVersion !== currentVersion + ? [{ elementId: operation.elementId, expected: operation.baseVersion, actual: currentVersion }] + : []; + }); const cells = mergeWorkbookCells( observedCellsFromSnapshot(snapshot), confirmed.map((cell) => observedCell(sheet, cell.id, cell.value, cell.version)), @@ -960,18 +969,33 @@ const VERIFY_WORKBOOK_TOOL: AgentTool = { const candidate = a.afterWrite === false ? undefined : verifyWorkbookValues({ cells, checks: checksForWorkbookOperations(operations) }); - const status = plan.status === "passed" && (!candidate || candidate.status === "passed") + const versionCoveragePassed = a.afterWrite !== false + || (targetsWithoutVersions.length === 0 && targetsWithStaleVersions.length === 0); + const status = plan.status === "passed" && (!candidate || candidate.status === "passed") && versionCoveragePassed ? "passed" as const : "needs_repair" as const; + const approvedOperations = a.afterWrite === false && status === "passed" + ? a.operations.map((operation) => ({ + ...operation, + baseVersion: targetVersions.get(operation.elementId)!, + })) + : undefined; return { ok: status === "passed", status, artifactId: sheet, phase: a.afterWrite === false ? "preflight" : "post_write", plan, + ...(approvedOperations ? { approvedOperations } : {}), ...(candidate ? { candidate } : {}), repairPrompt: [ ...plan.issues.map((issue) => `${issue.kind}: ${issue.repair}`), + ...(targetsWithoutVersions.length > 0 + ? [`target_version_unavailable: Re-read ${targetsWithoutVersions.join(", ")} before approving this write plan.`] + : []), + ...(targetsWithStaleVersions.length > 0 + ? [`stale_target_version: Re-inspect ${targetsWithStaleVersions.map((target) => `${target.elementId} (expected ${target.expected}, actual ${target.actual})`).join(", ")} before approving this write plan.`] + : []), ...(candidate?.repairPrompt ? [candidate.repairPrompt] : []), ].join("\n") || undefined, }; diff --git a/src/ui/RoomShell.tsx b/src/ui/RoomShell.tsx index f5b41849..1cd43797 100644 --- a/src/ui/RoomShell.tsx +++ b/src/ui/RoomShell.tsx @@ -7,7 +7,7 @@ import { useEffect, useRef, useState, type CSSProperties, type ReactNode } from "react"; import { PanelLeft, Table2, PanelRight, Moon, Sun, LogOut, ShieldCheck, X, HelpCircle, Copy, Check, MessageCircle, Sparkles, SlidersHorizontal, Palette, Gauge, Play, ChevronLeft, ChevronRight, Crosshair, WifiOff } from "lucide-react"; -import { useStore, type ActorProof } from "../app/store"; +import { useStore, type AgentCostKind, type ActorProof } from "../app/store"; import { OFFLINE_QUEUE_MAX } from "../notifications/offlineQueue"; import { Chat } from "./Chat"; import { Artifact } from "./panels/Artifact"; @@ -43,6 +43,10 @@ export function roomIntroSafetyCopy(mode: "memory" | "convex"): string { : "This live room uses the production backend: room state, edits, traces, and approvals persist for collaborators."; } +export function formatAgentCost(costUsd: number, costKind?: AgentCostKind): string { + return `${costKind === "exact" ? "" : "≈"}$${costUsd.toFixed(3)}`; +} + export function preferredRoomArtifact(arts: T[]): T | undefined { const scaleResearch = arts.find((a) => a.kind === "sheet" && @@ -1009,7 +1013,7 @@ function SignalStatusStrip({ ? [ { k: "Agents", v: `${sessions.length} active` }, { k: "Eval", v: run ? `${run.model} | ${run.toolCalls} tools` : "running" }, - { k: "Cost", v: run ? `$${run.costUsd.toFixed(3)}` : job ? job.modelPolicy : "-" }, + { k: "Cost", v: run ? formatAgentCost(run.costUsd, run.costKind) : job ? job.modelPolicy : "-" }, ] : []), ]; diff --git a/tests/agentJobsRuntime.test.ts b/tests/agentJobsRuntime.test.ts index 166e30e0..8c45ddd2 100644 --- a/tests/agentJobsRuntime.test.ts +++ b/tests/agentJobsRuntime.test.ts @@ -321,7 +321,7 @@ describe("agentJobs runtime contract", () => { const detail = await t.query(api.agentJobs.detail, { jobId: benchmark.jobId, requester: proof }); expect(detail?.job).toMatchObject({ runtimeProfile: "benchmark_completion", - maxAttempts: 100, + maxAttempts: 12, }); expect(detail?.job.request).toMatchObject({ runtimeProfile: "benchmark_completion", @@ -330,7 +330,237 @@ describe("agentJobs runtime contract", () => { const claimed = await t.mutation(internal.agentJobs.claimSlice, { jobId: benchmark.jobId, leaseId: "lease-benchmark-profile", leaseMs: 60_000 }); expect(claimed).toMatchObject({ runtimeProfile: "benchmark_completion", - maxAttempts: 100, + maxAttempts: 12, + }); + }); + + it("keeps public benchmark claims and cumulative retries within 12 attempts", async () => { + vi.useFakeTimers(); + try { + const { t, proof, roomId, artifactId } = await setupRoom({ seedElement: true }); + const started = await t.mutation(api.agentJobs.startPublicAsk, { + roomId, + requester: proof, + goal: "Run the official benchmark and complete the visible workbook", + contextArtifactId: String(artifactId), + routePolicy: "fast_default" as const, + runtimeProfile: "benchmark_completion" as const, + maxAttempts: 1_000, + }); + + // Emulate a queued row written before the production ceiling was lowered. + await t.run((ctx) => ctx.db.patch(started.jobId, { status: "queued", attempts: 10, maxAttempts: 1_000 })); + const claimed = await t.mutation(internal.agentJobs.claimSlice, { + jobId: started.jobId, + leaseId: "lease-benchmark-durable-cap", + leaseMs: 60_000, + }); + expect(claimed).toMatchObject({ attempt: 11, maxAttempts: 12 }); + + await t.run((ctx) => ctx.db.patch(started.jobId, { + status: "failed", + attempts: 11, + leaseId: "", + leaseUntil: 0, + })); + const finalRetry = await t.mutation(api.agentJobs.retry, { + jobId: started.jobId, + requester: proof, + additionalAttempts: 50, + }); + expect(finalRetry).toMatchObject({ ok: true, maxAttempts: 12 }); + + await t.run((ctx) => ctx.db.patch(started.jobId, { status: "failed", attempts: 12 })); + const exhaustedRetry = await t.mutation(api.agentJobs.retry, { + jobId: started.jobId, + requester: proof, + additionalAttempts: 50, + }); + expect(exhaustedRetry).toEqual({ ok: false, reason: "attempt_limit_reached", maxAttempts: 12 }); + + await t.run((ctx) => ctx.db.patch(started.jobId, { status: "queued", maxAttempts: 1_000 })); + const overCeilingClaim = await t.mutation(internal.agentJobs.claimSlice, { + jobId: started.jobId, + leaseId: "lease-benchmark-over-ceiling", + leaseMs: 60_000, + }); + expect(overCeilingClaim).toBeNull(); + + const detail = await t.query(api.agentJobs.detail, { jobId: started.jobId, requester: proof }); + expect(detail?.job).toMatchObject({ + status: "failed", + attempts: 12, + maxAttempts: 12, + error: "max_attempts_exhausted", + }); + } finally { + vi.useRealTimers(); + } + }); + + it("caps legacy public benchmark rows using entrypoint and goal fallbacks", async () => { + vi.useFakeTimers(); + try { + const { t, proof, roomId, artifactId } = await setupRoom({ seedElement: true }); + const goal = "Run the official SpreadsheetBench workbook benchmark"; + const started = await t.mutation(api.agentJobs.startPublicAsk, { + roomId, + requester: proof, + goal, + contextArtifactId: String(artifactId), + routePolicy: "fast_default" as const, + runtimeProfile: "benchmark_completion" as const, + maxAttempts: 1_000, + }); + + await t.run((ctx) => ctx.db.patch(started.jobId, { + status: "failed", + attempts: 12, + maxAttempts: 1_000, + scope: undefined, + runtimeProfile: undefined, + request: { commandText: goal, entrypoint: "public_ask" }, + })); + const retry = await t.mutation(api.agentJobs.retry, { + jobId: started.jobId, + requester: proof, + additionalAttempts: 50, + }); + expect(retry).toEqual({ ok: false, reason: "attempt_limit_reached", maxAttempts: 12 }); + + await t.run((ctx) => ctx.db.patch(started.jobId, { status: "queued", maxAttempts: 1_000 })); + const claim = await t.mutation(internal.agentJobs.claimSlice, { + jobId: started.jobId, + leaseId: "lease-legacy-benchmark-over-ceiling", + leaseMs: 60_000, + }); + expect(claim).toBeNull(); + const detail = await t.query(api.agentJobs.detail, { jobId: started.jobId, requester: proof }); + expect(detail?.job).toMatchObject({ status: "failed", attempts: 12, maxAttempts: 12, error: "max_attempts_exhausted" }); + } finally { + vi.useRealTimers(); + } + }); + + it("normalizes a reusable untagged public benchmark row at admission", async () => { + vi.useFakeTimers(); + try { + const { t, proof, roomId, artifactId } = await setupRoom({ seedElement: true }); + const args = { + roomId, + requester: proof, + goal: "Complete the held-out SpreadsheetBench workbook", + contextArtifactId: String(artifactId), + routePolicy: "fast_default" as const, + runtimeProfile: "benchmark_completion" as const, + maxAttempts: 1_000, + }; + const started = await t.mutation(api.agentJobs.startPublicAsk, args); + await t.run((ctx) => ctx.db.patch(started.jobId, { + status: "queued", + attempts: 5, + maxAttempts: 1_000, + scope: undefined, + runtimeProfile: undefined, + entrypoint: undefined, + request: undefined, + })); + + const reused = await t.mutation(api.agentJobs.startPublicAsk, args); + expect(reused).toMatchObject({ jobId: started.jobId, reused: true }); + const detail = await t.query(api.agentJobs.detail, { jobId: started.jobId, requester: proof }); + expect(detail?.job).toMatchObject({ + scope: "public_room", + runtimeProfile: "benchmark_completion", + entrypoint: "public_ask", + attempts: 5, + maxAttempts: 12, + }); + } finally { + vi.useRealTimers(); + } + }); + + it("caps the oldest fully untagged free-auto benchmark rows during retry and claim", async () => { + vi.useFakeTimers(); + try { + const { t, proof, roomId, artifactId } = await setupRoom({ seedElement: true }); + const started = await t.mutation(api.agentJobs.startPublicAsk, { + roomId, + requester: proof, + goal: "Complete the official SpreadsheetBench workbook", + contextArtifactId: String(artifactId), + routePolicy: "fast_default" as const, + runtimeProfile: "benchmark_completion" as const, + maxAttempts: 1_000, + }); + + // Reproduce the original startFreeAuto row shape before scope, entrypoint, request, and + // runtimeProfile were persisted. This path intentionally does not pass through re-admission. + await t.run((ctx) => ctx.db.patch(started.jobId, { + status: "failed", + attempts: 12, + maxAttempts: 1_000, + scope: undefined, + runtimeProfile: undefined, + entrypoint: undefined, + request: undefined, + modelPolicy: "openrouter/free-auto", + runtime: "workflow", + })); + const retry = await t.mutation(api.agentJobs.retry, { + jobId: started.jobId, + requester: proof, + additionalAttempts: 50, + }); + expect(retry).toEqual({ ok: false, reason: "attempt_limit_reached", maxAttempts: 12 }); + + await t.run((ctx) => ctx.db.patch(started.jobId, { status: "queued", maxAttempts: 1_000 })); + const claimed = await t.mutation(internal.agentJobs.claimSlice, { + jobId: started.jobId, + leaseId: "lease-oldest-legacy-benchmark-over-ceiling", + leaseMs: 60_000, + }); + expect(claimed).toBeNull(); + const detail = await t.query(api.agentJobs.detail, { jobId: started.jobId, requester: proof }); + expect(detail?.job).toMatchObject({ + status: "failed", + attempts: 12, + maxAttempts: 12, + error: "max_attempts_exhausted", + }); + } finally { + vi.useRealTimers(); + } + }); + + it("caps public inline benchmark jobs while preserving the existing non-public inline bound", async () => { + const { t, proof, roomId, artifactId } = await setupRoom(); + const publicJob = await t.mutation(api.agentJobs.createOrReuse, { + ...jobArgs({ roomId, artifactId, proof, idempotencyKey: "job-runtime-public-benchmark-inline" }), + runtimeProfile: "benchmark_completion" as const, + maxAttempts: 20, + }); + const nonPublicJob = await t.mutation(api.agentJobs.createOrReuse, { + ...jobArgs({ roomId, artifactId, proof, idempotencyKey: "job-runtime-private-benchmark-inline" }), + scope: "private_user" as const, + runtimeProfile: "benchmark_completion" as const, + maxAttempts: 20, + }); + + const publicDetail = await t.query(api.agentJobs.detail, { jobId: publicJob.jobId, requester: proof }); + const nonPublicDetail = await t.query(api.agentJobs.detail, { jobId: nonPublicJob.jobId, requester: proof }); + expect(publicDetail?.job).toMatchObject({ + scope: "public_room", + runtime: "inline", + runtimeProfile: "benchmark_completion", + maxAttempts: 12, + }); + expect(nonPublicDetail?.job).toMatchObject({ + scope: "private_user", + runtime: "inline", + runtimeProfile: "benchmark_completion", + maxAttempts: 20, }); }); @@ -348,7 +578,7 @@ describe("agentJobs runtime contract", () => { const detail = await t.query(api.agentJobs.detail, { jobId: started.jobId, requester: proof }); expect(detail?.job).toMatchObject({ runtimeProfile: "benchmark_completion", - maxAttempts: 1000, + maxAttempts: 12, }); expect(detail?.job.request).toMatchObject({ runtimeProfile: "benchmark_completion", @@ -357,7 +587,7 @@ describe("agentJobs runtime contract", () => { const claimed = await t.mutation(internal.agentJobs.claimSlice, { jobId: started.jobId, leaseId: "lease-benchmark-inferred", leaseMs: 60_000 }); expect(claimed).toMatchObject({ runtimeProfile: "benchmark_completion", - maxAttempts: 1000, + maxAttempts: 12, }); }); @@ -739,6 +969,14 @@ describe("agentJobs runtime contract", () => { it("finishInteractive writes attempts, operation events, and materialized counters", async () => { const { t, proof, roomId, artifactId } = await setupRoom(); const { jobId } = await t.mutation(api.agentJobs.createOrReuse, jobArgs({ roomId, artifactId, proof, idempotencyKey: "job-runtime-finish" })); + await t.mutation(internal.agentJobs.recordLiveOperation, { + jobId, + sequence: 1_000, + kind: "model_call", + name: "test-model", + status: "started", + countDelta: 0, + }); await t.mutation(internal.agentJobs.finishInteractive, { jobId, @@ -749,8 +987,11 @@ describe("agentJobs runtime contract", () => { ms: 1200, inputTokens: 100, outputTokens: 25, + cachedInputTokens: 40, + cacheCreationInputTokens: 10, costUsd: 0.001, - modelCalls: 1, + costKind: "exact" as const, + modelCalls: 2, toolCalls: 2, queryCount: 3, mutationCount: 4, @@ -765,6 +1006,132 @@ describe("agentJobs runtime contract", () => { expect(detail?.job.queryCount).toBe(3); expect(detail?.job.mutationCount).toBe(5); expect(detail?.job.receiptCount).toBe(1); + expect(detail?.job).toMatchObject({ + modelCallCount: 2, + toolCallCount: 2, + inputTokens: 100, + outputTokens: 25, + cachedInputTokens: 40, + cacheCreationInputTokens: 10, + costUsd: 0.001, + costKind: "exact", + }); + expect(detail?.attempts[0]).toMatchObject({ + modelCalls: 2, + toolCalls: 2, + cachedInputTokens: 40, + cacheCreationInputTokens: 10, + costKind: "exact", + }); + expect(detail?.operations + .filter((event) => event.kind === "model_call") + .reduce((sum, event) => sum + (event.countDelta ?? 1), 0)).toBe(2); + }); + + it("finishInteractive reconstructs missing aggregate fields from prior attempts", async () => { + const { t, proof, roomId, artifactId } = await setupRoom(); + const { jobId } = await t.mutation(api.agentJobs.createOrReuse, jobArgs({ roomId, artifactId, proof, idempotencyKey: "job-runtime-interactive-aggregate-migration" })); + const now = Date.now(); + await t.run(async (ctx) => { + await ctx.db.insert("agentJobAttempts", { + jobId, + attempt: 1, + status: "handoff", + resolvedModel: "legacy-model", + stopReason: "time_budget", + ms: 500, + inputTokens: 80, + outputTokens: 20, + cachedInputTokens: 30, + cacheCreationInputTokens: 5, + costUsd: 0.002, + modelCalls: 3, + toolCalls: 4, + startedAt: now - 500, + endedAt: now, + }); + await ctx.db.patch(jobId, { + attempts: 1, + modelCallCount: undefined, + toolCallCount: undefined, + inputTokens: undefined, + outputTokens: undefined, + cachedInputTokens: undefined, + cacheCreationInputTokens: undefined, + costUsd: undefined, + costKind: undefined, + }); + }); + + await t.mutation(internal.agentJobs.finishInteractive, { + jobId, + status: "completed", + finalText: "done after deploy", + resolvedModel: "current-model", + stopReason: "done", + ms: 100, + inputTokens: 20, + outputTokens: 5, + cachedInputTokens: 10, + cacheCreationInputTokens: 2, + costUsd: 0.001, + costKind: "exact" as const, + modelCalls: 2, + toolCalls: 1, + }); + + const detail = await t.query(api.agentJobs.detail, { jobId, requester: proof }); + expect(detail?.attempts).toHaveLength(2); + expect(detail?.job).toMatchObject({ + modelCallCount: 5, + toolCallCount: 5, + inputTokens: 100, + outputTokens: 25, + cachedInputTokens: 40, + cacheCreationInputTokens: 7, + costKind: "estimated", + }); + expect(detail?.job.costUsd).toBeCloseTo(0.003); + }); + + it("does not start a continuation workflow for a terminal interactive protocol stall", async () => { + const { t, proof, roomId, artifactId } = await setupRoom(); + const { jobId } = await t.mutation(api.agentJobs.createOrReuse, jobArgs({ + roomId, + artifactId, + proof, + idempotencyKey: "job-runtime-protocol-stall-terminal", + })); + + await t.mutation(internal.agentJobs.finishInteractive, { + jobId, + status: "failed", + finalText: "Required tool call missing after four turns.", + error: "protocol_stall", + handoff: { terminalReason: "protocol_stall" }, + scheduleWorkflow: true, + resolvedModel: "test-model", + stopReason: "step_budget", + ms: 100, + inputTokens: 40, + outputTokens: 20, + cachedInputTokens: 0, + cacheCreationInputTokens: 0, + costUsd: 0.001, + costKind: "exact" as const, + modelCalls: 4, + toolCalls: 0, + }); + + const detail = await t.query(api.agentJobs.detail, { jobId, requester: proof }); + expect(detail?.job).toMatchObject({ + status: "failed", + error: "protocol_stall", + modelCallCount: 4, + }); + expect(detail?.job.workflowId).toBeUndefined(); + expect(detail?.job.nextRunAt).toBe(0); + expect(detail?.operations.map((event) => event.kind)).not.toContain("scheduler"); }); it("records live operation events before an interactive run reaches a terminal state", async () => { @@ -785,7 +1152,7 @@ describe("agentJobs runtime contract", () => { kind: "model_call", name: "deepseek/deepseek-v4-flash", status: "started", - countDelta: 1, + countDelta: 0, }); const read = await t.mutation(internal.agentJobs.recordLiveOperation, { jobId, @@ -803,6 +1170,7 @@ describe("agentJobs runtime contract", () => { expect(read).toEqual({ ok: true }); expect(detail?.job.status).toBe("running"); expect(detail?.operations.map((event) => event.name)).toEqual(expect.arrayContaining(["deepseek/deepseek-v4-flash", "read_range"])); + expect(detail?.operations.find((event) => event.name === "deepseek/deepseek-v4-flash")?.countDelta).toBe(0); expect(detail?.operations.find((event) => event.name === "read_range")?.affectedIds).toEqual(["row1__variance"]); }); @@ -1031,6 +1399,189 @@ describe("agentJobs runtime contract", () => { expect(detail?.reasoningFrames.map((frame) => frame.status)).toEqual(["completed", "completed", "completed"]); }); + it("finishSlice preserves failover call counts and cache cost provenance", async () => { + const { t, proof, roomId, artifactId } = await setupRoom(); + const { jobId } = await t.mutation(api.agentJobs.createOrReuse, jobArgs({ roomId, artifactId, proof, idempotencyKey: "job-runtime-exact-accounting" })); + await seedRuntimeReasoningFrames(t, { roomId, artifactId, jobId }); + await t.mutation(internal.agentJobs.claimSlice, { jobId, leaseId: "lease-accounting", leaseMs: 60_000 }); + + const finished = await t.mutation(internal.agentJobs.finishSlice, { + ...finishSliceArgs({ jobId, leaseId: "lease-accounting", attempt: 1 }), + modelCalls: 2, + toolCalls: 3, + inputTokens: 120, + outputTokens: 30, + cachedInputTokens: 70, + cacheCreationInputTokens: 11, + costUsd: 0.004, + costKind: "estimated" as const, + }); + + expect(finished).toEqual({ ok: true }); + const detail = await t.query(api.agentJobs.detail, { jobId, requester: proof }); + expect(detail?.job).toMatchObject({ + modelCallCount: 2, + toolCallCount: 3, + inputTokens: 120, + outputTokens: 30, + cachedInputTokens: 70, + cacheCreationInputTokens: 11, + costUsd: 0.004, + costKind: "estimated", + }); + expect(detail?.attempts[0]).toMatchObject({ + modelCalls: 2, + toolCalls: 3, + cachedInputTokens: 70, + cacheCreationInputTokens: 11, + costKind: "estimated", + }); + + const secondClaim = await t.mutation(internal.agentJobs.claimSlice, { jobId, leaseId: "lease-accounting-2", leaseMs: 60_000 }); + expect(secondClaim?.attempt).toBe(2); + await t.mutation(internal.agentJobs.finishSlice, { + ...finishSliceArgs({ jobId, leaseId: "lease-accounting-2", attempt: 2 }), + modelCalls: 1, + toolCalls: 2, + inputTokens: 20, + outputTokens: 5, + cachedInputTokens: 3, + cacheCreationInputTokens: 2, + costUsd: 0.001, + costKind: "exact" as const, + }); + + const completed = await t.query(api.agentJobs.detail, { jobId, requester: proof }); + expect(completed?.job).toMatchObject({ + modelCallCount: 3, + toolCallCount: 5, + inputTokens: 140, + outputTokens: 35, + cachedInputTokens: 73, + cacheCreationInputTokens: 13, + costKind: "estimated", + }); + expect(completed?.job.costUsd).toBeCloseTo(0.005); + }); + + it("finishSlice reconstructs a legacy attempt baseline before adding the current slice", async () => { + const { t, proof, roomId, artifactId } = await setupRoom(); + const { jobId } = await t.mutation(api.agentJobs.createOrReuse, jobArgs({ roomId, artifactId, proof, idempotencyKey: "job-runtime-slice-aggregate-migration" })); + await seedRuntimeReasoningFrames(t, { roomId, artifactId, jobId }); + const now = Date.now(); + await t.run(async (ctx) => { + await ctx.db.insert("agentJobAttempts", { + jobId, + attempt: 1, + status: "handoff", + resolvedModel: "legacy-model", + stopReason: "time_budget", + ms: 400, + inputTokens: 50, + outputTokens: 10, + costUsd: 0.002, + startedAt: now - 400, + endedAt: now, + }); + await ctx.db.patch(jobId, { + attempts: 1, + modelCallCount: undefined, + toolCallCount: undefined, + inputTokens: undefined, + outputTokens: undefined, + cachedInputTokens: undefined, + cacheCreationInputTokens: undefined, + costUsd: undefined, + costKind: undefined, + }); + }); + const claimed = await t.mutation(internal.agentJobs.claimSlice, { jobId, leaseId: "lease-legacy-accounting", leaseMs: 60_000 }); + expect(claimed?.attempt).toBe(2); + + await t.mutation(internal.agentJobs.finishSlice, { + ...finishSliceArgs({ jobId, leaseId: "lease-legacy-accounting", attempt: 2 }), + modelCalls: 2, + toolCalls: 3, + inputTokens: 25, + outputTokens: 5, + cachedInputTokens: 10, + cacheCreationInputTokens: 4, + costUsd: 0.001, + costKind: "exact" as const, + }); + + const detail = await t.query(api.agentJobs.detail, { jobId, requester: proof }); + expect(detail?.attempts).toHaveLength(2); + expect(detail?.job).toMatchObject({ + modelCallCount: 3, + toolCallCount: 4, + inputTokens: 75, + outputTokens: 15, + cachedInputTokens: 10, + cacheCreationInputTokens: 4, + costKind: "estimated", + }); + expect(detail?.job.costUsd).toBeCloseTo(0.003); + }); + + it("finishSlice keeps an explicit zero-provider-call slice at zero", async () => { + const { t, proof, roomId, artifactId } = await setupRoom(); + const { jobId } = await t.mutation(api.agentJobs.createOrReuse, jobArgs({ roomId, artifactId, proof, idempotencyKey: "job-runtime-zero-model-calls" })); + await seedRuntimeReasoningFrames(t, { roomId, artifactId, jobId }); + await t.mutation(internal.agentJobs.claimSlice, { jobId, leaseId: "lease-zero-calls", leaseMs: 60_000 }); + + await t.mutation(internal.agentJobs.finishSlice, { + ...finishSliceArgs({ jobId, leaseId: "lease-zero-calls", attempt: 1 }), + modelCalls: 0, + toolCalls: 0, + inputTokens: 0, + outputTokens: 0, + cachedInputTokens: 0, + cacheCreationInputTokens: 0, + costUsd: 0, + costKind: "exact" as const, + }); + + const detail = await t.query(api.agentJobs.detail, { jobId, requester: proof }); + expect(detail?.job.modelCallCount).toBe(0); + expect(detail?.job.toolCallCount).toBe(0); + expect(detail?.attempts[0]).toMatchObject({ modelCalls: 0, toolCalls: 0, costKind: "exact" }); + }); + + it("keeps the deterministic benchmark executor at zero provider calls", async () => { + const { t, proof, roomId, artifactId } = await setupRoom(); + const { jobId } = await t.mutation(api.agentJobs.createOrReuse, jobArgs({ + roomId, + artifactId, + proof, + idempotencyKey: "job-runtime-deterministic-zero-model-calls", + })); + await seedRuntimeReasoningFrames(t, { roomId, artifactId, jobId }); + await t.mutation(internal.agentJobs.claimSlice, { jobId, leaseId: "lease-deterministic", leaseMs: 60_000 }); + const runId = await t.mutation(internal.agentRuns.claim, { + jobId, + roomId, + agentId: "agent_room", + model: "deterministic/hmda-underwriting", + goal: "Complete the deterministic HMDA benchmark.", + }); + + await t.mutation(internal.agentJobs.completeDeterministicBenchmarkSlice, { + jobId, + leaseId: "lease-deterministic", + runId, + finalText: "done", + resolvedModel: "deterministic/hmda-underwriting", + }); + + const detail = await t.query(api.agentJobs.detail, { jobId, requester: proof }); + expect(detail?.job.status).toBe("completed"); + expect(detail?.job.modelCallCount).toBe(0); + expect(detail?.operations + .filter((event) => event.kind === "model_call") + .reduce((sum, event) => sum + (event.countDelta ?? 1), 0)).toBe(0); + }); + it("finishSlice compacts oversized handoff and cursor payloads before patching the job row", async () => { const { t, proof, roomId, artifactId } = await setupRoom(); const { jobId } = await t.mutation(api.agentJobs.createOrReuse, jobArgs({ roomId, artifactId, proof, idempotencyKey: "job-runtime-compact-continuation" })); @@ -1531,5 +2082,8 @@ function finishSliceArgs(args: { jobId: Id<"agentJobs">; leaseId: string; attemp inputTokens: 10, outputTokens: 5, costUsd: 0.0001, + costKind: "exact" as const, + modelCalls: 1, + toolCalls: 1, }; } diff --git a/tests/agentJobsSource.test.ts b/tests/agentJobsSource.test.ts index 5386e2eb..19856036 100644 --- a/tests/agentJobsSource.test.ts +++ b/tests/agentJobsSource.test.ts @@ -108,8 +108,9 @@ describe("long-running agent job source invariants", () => { expect(jobs).toContain("function inferredRuntimeProfileForGoal"); expect(jobs).toContain("runtimeProfile = a.runtimeProfile ?? inferredRuntimeProfileForGoal(a.goal)"); expect(jobs).toContain("runtimeProfile: job.runtimeProfile"); - expect(jobs).toContain('defaultMaxAttempts = runtimeProfile === "benchmark_completion" ? 1000'); + expect(jobs).toContain("const PUBLIC_BENCHMARK_COMPLETION_MAX_ATTEMPTS = 12"); expect(runner).toContain("function maxStepsForJob"); + expect(runner).toContain('result.handoff?.terminalReason === "protocol_stall"'); expect(runner).toContain("BENCHMARK_AGENT_MAX_STEPS_PER_SLICE"); expect(runner).toContain("FREE_AUTO_JOB_MAX_STEPS_PER_SLICE"); expect(runner).toContain("defaultMaxStepsForEntrypoint(entrypoint), 1, 256"); @@ -118,9 +119,25 @@ describe("long-running agent job source invariants", () => { expect(store).toContain("noderoom.nodeagentRuntimeProfile"); expect(store).toContain('focusMode === "1" || focusMode === "true"'); expect(store).toContain("maxAttemptsForRuntimeProfile"); + expect(store).toContain("Math.min(requested ?? PUBLIC_BENCHMARK_COMPLETION_MAX_ATTEMPTS, PUBLIC_BENCHMARK_COMPLETION_MAX_ATTEMPTS)"); expect(spec).toContain('window.localStorage.setItem("noderoom.nodeagentRuntimeProfile", "benchmark_completion")'); }); + it("hard-caps public benchmark attempts at admission, claim, retry, and browser boundaries", () => { + const jobs = readFileSync("convex/agentJobs.ts", "utf8"); + const store = readFileSync("src/app/store.tsx", "utf8"); + + expect(jobs).toContain('entrypoint === "public_ask" || entrypoint === "free" || entrypoint === "room_work"'); + expect(jobs).toContain('inferredRuntimeProfileForGoal(goal) === "benchmark_completion"'); + expect(jobs).toContain("const reusableIdentity = {"); + expect(jobs).toContain("durableMaxAttemptsForJob(job, requestedMaxAttempts)"); + expect(jobs).toContain("durableMaxAttemptsForJob(job, job.maxAttempts)"); + expect(jobs).toContain('reason: "attempt_limit_reached"'); + expect(jobs).toContain('error: "max_attempts_exhausted"'); + expect(jobs).not.toContain('runtimeProfile === "benchmark_completion" ? 1000'); + expect(store).not.toContain("Math.min(requested ?? 12, 50)"); + }); + it("keeps /ask model policy during workflow handoff while allowing /free overrides", () => { const runner = readFileSync("convex/agentJobRunner.ts", "utf8"); const jobs = readFileSync("convex/agentJobs.ts", "utf8"); @@ -128,7 +145,11 @@ describe("long-running agent job source invariants", () => { expect(runner).toContain('modelPolicy === "openrouter/free-auto"'); expect(runner).toContain("process.env.FREE_AUTO_JOB_MODEL ?? modelPolicy"); - expect(runner).toContain("const model = agentModel(resolvedModelPolicy, { entrypoint })"); + expect(runner).toContain("const model = agentModel(resolvedModelPolicy, {"); + expect(runner).toContain("fallbackModelIds: fallbackModelsForRoute(resolvedModelPolicy)"); + expect(runner).toContain('claimed.routePolicy === "explicit"'); + expect(runner).toContain("authorizedModelForFramePhase"); + expect(runner).toContain("artifacts: egressArtifacts"); expect(runner).toContain("function runnerEntrypoint"); expect(runner).toContain("defaultMaxStepsForEntrypoint(entrypoint)"); expect(jobs).toContain("artifactMeta: art.meta"); @@ -138,6 +159,22 @@ describe("long-running agent job source invariants", () => { expect(model).toContain("candidateSignal, 0"); }); + it("checkpoints concrete route health and never reschedules a terminal protocol stall", () => { + const agent = readFileSync("convex/agent.ts", "utf8"); + const runner = readFileSync("convex/agentJobRunner.ts", "utf8"); + const model = readFileSync("src/nodeagent/models/convexModel.ts", "utf8"); + + expect(model).toContain("routeState() {"); + expect(model).toContain("hydrateConcreteRouteState"); + expect(runner).toContain("routeState: routeStateFromCursor(claimed.cursor)"); + expect(runner).toContain("modelRouteState: result.modelRouteState"); + expect(runner).toContain("done || nonResumable ? undefined : await checkpoint"); + expect(agent).toContain('const protocolStall = result.handoff?.terminalReason === "protocol_stall"'); + expect(agent).toContain("const terminal = done || protocolStall"); + expect(agent).toContain("scheduleWorkflow: !terminal"); + expect(agent).toContain('status: done ? "completed" : protocolStall ? "failed" : "paused"'); + }); + it("enforces provider route receipts and private-stream egress gates", () => { const model = readFileSync("src/nodeagent/models/convexModel.ts", "utf8"); const streaming = readFileSync("convex/streaming.ts", "utf8"); diff --git a/tests/agentRuntime.test.ts b/tests/agentRuntime.test.ts index 18175e93..2698457a 100644 --- a/tests/agentRuntime.test.ts +++ b/tests/agentRuntime.test.ts @@ -250,7 +250,7 @@ describe("agent runtime — collaboration under concurrency", () => { expect(r.exhausted).toBe(true); expect(r.stopReason).toBe("step_budget"); - expect(r.handoff?.summary).toContain("required tool call missing"); + expect(r.handoff?.summary).toContain("2/4 required-tool misses"); expect(r.trace.at(-1)?.tool).toBe("handoff"); }); @@ -506,8 +506,10 @@ describe("agent runtime — collaboration under concurrency", () => { expect(r.exhausted).toBe(true); expect(r.stopReason).toBe("step_budget"); expect(r.messages.some((message) => - message.role === "user" && !!message.content?.includes("BTB deliverable package"), + message.role === "user" && !!message.content?.includes("tool_required_no_call"), )).toBe(true); + expect(r.handoff?.terminalReason).toBeUndefined(); + expect(r.handoff?.summary).toContain("2/4 required-tool misses"); }); it("requires provider tool calls while a BTB package is still missing", async () => { @@ -584,7 +586,7 @@ describe("agent runtime — collaboration under concurrency", () => { expect(r.trace.some((event) => event.tool === "create_btb_deliverable_package")).toBe(true); }); - it("checkpoints repeated no-tool BTB provider output without exposing a raw terminal marker", async () => { + it("terminates repeated no-tool BTB provider output without exposing a raw terminal marker", async () => { const { rt } = setup(); const choices: Array = []; const packageTool: AgentTool = { @@ -620,9 +622,156 @@ describe("agent runtime — collaboration under concurrency", () => { expect(choices.length).toBe(4); expect(r.exhausted).toBe(true); expect(r.stopReason).toBe("step_budget"); - expect(r.handoff?.summary).toContain("required tool call missing"); - expect(r.handoff?.summary).toContain("checkpointed"); + expect(r.handoff?.terminalReason).toBe("protocol_stall"); + expect(r.handoff?.summary).toContain("stopped"); + expect(r.handoff?.summary).not.toContain("checkpointed"); expect(r.handoff?.summary).not.toContain("tool_required_no_call_terminal"); + expect(r.trace.at(-1)?.result).toMatchObject({ + terminalReason: "protocol_stall", + remainingToolCalls: [], + }); + }); + + it("carries required-tool misses across one-step slices and terminates exactly on the fourth provider call", async () => { + const { rt } = setup(); + const packageTool: AgentTool = { + name: "create_btb_deliverable_package", + description: "Create the final BTB package.", + schema: z.object({}), + execute: async () => ({ ok: true }), + }; + let providerCalls = 0; + const model: AgentModel = { + name: "durable-no-tool-provider", + async next() { + providerCalls += 1; + return { + text: "I will call the package tool next.", + toolCalls: [], + done: true, + usage: { inputTokens: 3, outputTokens: 2, modelCalls: 1 }, + }; + }, + }; + let initialMessages: AgentMessage[] | undefined; + let result: Awaited> | undefined; + let persistedModelCalls = 0; + for (let slice = 1; slice <= 4; slice += 1) { + result = await runAgent({ + rt, + goal: "Run official BankerToolBench task btb-test and create deliverable package", + model, + tools: [...ROOM_TOOLS, packageTool], + maxSteps: 1, + initialMessages, + }); + persistedModelCalls += result.usage.modelCalls; + initialMessages = result.messages; + if (slice < 4) { + expect(result.handoff?.terminalReason).toBeUndefined(); + expect(result.handoff?.summary).toContain(`${slice}/4 required-tool misses`); + } + } + + expect(providerCalls).toBe(4); + expect(persistedModelCalls).toBe(4); + expect(result?.handoff?.terminalReason).toBe("protocol_stall"); + expect(result?.handoff?.summary).toContain("after 4 required tool-use turns"); + }); + + it("terminates four identical blocked tool calls without checkpointing a queued call", async () => { + const { rt } = setup(); + let turns = 0; + let executions = 0; + const writeTool: AgentTool = { + name: "write_locked_cells", + description: "Write cells.", + schema: z.object({ ops: z.array(z.object({ elementId: z.string(), value: z.unknown() })) }), + execute: async () => { + executions += 1; + return { ok: true }; + }, + }; + const result = await runAgent({ + rt, + goal: "Fill the spreadsheet cells with the requested values.", + model: { + name: "blocked-tool-provider", + async next() { + turns += 1; + return { + toolCalls: [ + { id: `blocked-${turns}`, tool: "write_locked_cells", args: { ops: [{ elementId: "B2", value: 10 }] } }, + ...(turns === 4 + ? [{ id: "poisoned-blocked-call", tool: "write_locked_cells", args: { ops: [{ elementId: "B3", value: 20 }] } }] + : []), + ], + done: false, + }; + }, + }, + tools: [writeTool], + hooks: [{ + preTool: () => ({ + blocked: true as const, + reason: "verified_workbook_workflow:write_plan_mismatch: blocked", + failureKind: "evidence_required" as const, + }), + }], + maxSteps: 10, + }); + + expect(turns).toBe(4); + expect(executions).toBe(0); + expect(result.handoff?.terminalReason).toBe("protocol_stall"); + expect(result.handoff?.remainingToolCalls).toEqual([]); + expect(result.messages.flatMap((message) => message.toolCalls ?? []).map((call) => call.id)) + .not.toContain("poisoned-blocked-call"); + expect(result.trace.filter((event) => event.tool === "write_locked_cells")).toHaveLength(4); + }); + + it("terminates four identical tool argument errors without checkpointing a queued call", async () => { + const { rt } = setup(); + let turns = 0; + let executions = 0; + const strictTool: AgentTool = { + name: "strict_tool", + description: "Requires an id.", + schema: z.object({ id: z.string() }), + execute: async () => { + executions += 1; + return { ok: true }; + }, + }; + const result = await runAgent({ + rt, + goal: "Process the record with strict_tool.", + model: { + name: "invalid-argument-provider", + async next() { + turns += 1; + return { + toolCalls: [ + { id: `invalid-${turns}`, tool: "strict_tool", args: {} }, + ...(turns === 4 + ? [{ id: "poisoned-argument-call", tool: "strict_tool", args: { id: "must-not-replay" } }] + : []), + ], + done: false, + }; + }, + }, + tools: [strictTool], + maxSteps: 10, + }); + + expect(turns).toBe(4); + expect(executions).toBe(0); + expect(result.handoff?.terminalReason).toBe("protocol_stall"); + expect(result.handoff?.remainingToolCalls).toEqual([]); + expect(result.messages.flatMap((message) => message.toolCalls ?? []).map((call) => call.id)) + .not.toContain("poisoned-argument-call"); + expect(result.trace.filter((event) => event.tool === "strict_tool")).toHaveLength(4); }); it("does not count failed BTB package attempts as a completed package", async () => { @@ -982,6 +1131,52 @@ describe("agent runtime — collaboration under concurrency", () => { expect(sawResumeContract).toBe(true); }); + it("accounts for two provider failover calls and exact cost from one logical model turn", async () => { + const { rt } = setup(); + let logicalModelCalls = 0; + let fallbackPriceCalls = 0; + const model: AgentModel = { + name: "mixed-route-failover", + async next() { + logicalModelCalls += 1; + return { + text: "Summary complete.", + toolCalls: [], + done: true, + usage: { + inputTokens: 22, + outputTokens: 14, + cachedInputTokens: 3, + modelCalls: 2, + costUsd: 0.125, + }, + }; + }, + }; + + const result = await runAgent({ + rt, + goal: "summarize the current room", + model, + tools: [], + maxSteps: 1, + priceStep: () => { + fallbackPriceCalls += 1; + return 999; + }, + }); + + expect(logicalModelCalls).toBe(1); + expect(fallbackPriceCalls).toBe(0); + expect(result.usage).toMatchObject({ + inputTokens: 22, + outputTokens: 14, + cachedInputTokens: 3, + modelCalls: 2, + costUsd: 0.125, + }); + }); + it("time budget: stops with a resumable handoff before another model turn", async () => { const { rt } = setup(); let modelCalls = 0; @@ -1029,6 +1224,60 @@ describe("agent runtime — collaboration under concurrency", () => { expect(Date.now() - startedAt).toBeLessThan(1_000); expect(r.stopReason).toBe("time_budget"); expect(r.handoff?.reason).toBe("time_budget"); + expect(r.usage).toMatchObject({ modelCalls: 0, costKind: "estimated" }); + }); + + it("time budget waits briefly for provider usage and route cooldowns to settle", async () => { + const { rt } = setup(); + const cooldownUntil = Date.now() + 60_000; + const routeState = { preferredModelId: "fallback", cooldownUntil: {} as Record }; + const model: AgentModel = { + name: "abort-aware-provider", + routeState: () => routeState, + next: async ({ signal }) => new Promise((_resolve, reject) => { + signal?.addEventListener("abort", () => { + setTimeout(() => { + routeState.cooldownUntil.primary = cooldownUntil; + reject(Object.assign(new Error("provider aborted after reporting usage"), { + usage: { + inputTokens: 23, + outputTokens: 7, + cachedInputTokens: 5, + cacheCreationInputTokens: 2, + modelCalls: 1, + costUsd: 0.015, + costKind: "exact", + }, + })); + }, 5); + }, { once: true }); + }), + }; + + const result = await runAgent({ + rt, + goal: "capture partial provider usage before handoff", + model, + tools: ROOM_TOOLS, + maxSteps: 1, + deadlineAt: Date.now() + 30, + reserveMs: 0, + }); + + expect(result.stopReason).toBe("time_budget"); + expect(result.usage).toMatchObject({ + inputTokens: 23, + outputTokens: 7, + cachedInputTokens: 5, + cacheCreationInputTokens: 2, + modelCalls: 1, + costUsd: 0.015, + costKind: "exact", + }); + expect(result.modelRouteState).toMatchObject({ + preferredModelId: "fallback", + cooldownUntil: { primary: cooldownUntil }, + }); }); it("resumes a long-running job across multiple step-budget slices", async () => { @@ -1103,6 +1352,7 @@ describe("agent runtime — collaboration under concurrency", () => { expect(first.stopReason).toBe("time_budget"); expect(executed).toBe(1); + expect(first.handoff?.terminalReason).toBeUndefined(); expect(first.handoff?.remainingToolCalls.map((c) => c.args.id)).toEqual(["second"]); const resumed = await runAgent({ diff --git a/tests/convexCredits.test.ts b/tests/convexCredits.test.ts index a49022c5..25a367f6 100644 --- a/tests/convexCredits.test.ts +++ b/tests/convexCredits.test.ts @@ -38,7 +38,7 @@ async function readRoomCredits(t: ReturnType, roomId: any) { } describe("convex credits — grant + reserve + settle", () => { - it("grant seeds the balance; reserve holds; settle debits actual + refunds the rest", async () => { + it("grant seeds the balance; reserve holds; settle labels estimated spend and refunds the rest", async () => { const t = convexTest(schema, modules); const roomId = await seedRoom(t); await t.mutation(internal.credits.grantCredits, { roomId, credits: 20, source: "pilot" }); @@ -54,9 +54,12 @@ describe("convex credits — grant + reserve + settle", () => { // Settle cheaper than the hold → unused refunded, reserved released. const cheapUsd = estimateCostFor("standard").estimateUsd / 2; - const s = await t.mutation(internal.credits.settle, { roomId, reservationKey: "job_1", actualUsd: cheapUsd }); + const s = await t.mutation(internal.credits.settle, { roomId, reservationKey: "job_1", actualUsd: cheapUsd, costKind: "estimated" }); expect(s.ok).toBe(true); + expect((s as any).costKind).toBe("estimated"); expect((s as any).refundedCredits).toBeGreaterThan(0); + const ledger = await t.run(async (ctx) => ctx.db.query("creditLedger").withIndex("by_reservation", (q) => q.eq("reservationKey", "job_1")).collect()); + expect(ledger.find((row) => row.kind === "settle")?.costKind).toBe("estimated"); rc = await readRoomCredits(t, roomId); expect(rc?.reservedCredits).toBe(0); expect((rc?.availableCredits ?? 0) + (rc?.lifetimeSpentCredits ?? 0)).toBeCloseTo(20, 1); @@ -136,6 +139,8 @@ describe("convex credits — idempotency", () => { expect((b as any).idempotent).toBe(true); const rc = await readRoomCredits(t, roomId); expect(rc?.lifetimeSpentCredits).toBeLessThanOrEqual(QUICK_HOLD + 0.5); // not double-charged + const ledger = await t.run(async (ctx) => ctx.db.query("creditLedger").withIndex("by_reservation", (q) => q.eq("reservationKey", "once")).collect()); + expect(ledger.find((row) => row.kind === "settle")?.costKind).toBe("estimated"); }); it("settle for an unknown reservation is rejected honestly", async () => { @@ -191,7 +196,7 @@ describe("convex credits — overspend, pause, sweep", () => { expect(rc?.availableCredits).toBe(20); // fully refunded }); - it("COST-AWARE sweep: a finished-but-unsettled run is charged its ACTUAL cost, not fully refunded", async () => { + it("COST-AWARE sweep: a finished-but-unsettled run is charged its recorded cost, not fully refunded", async () => { const t = convexTest(schema, modules); const roomId = await seedRoom(t); const t0 = 2_000_000; @@ -201,7 +206,7 @@ describe("convex credits — overspend, pause, sweep", () => { await t.run(async (ctx: any) => { await ctx.db.insert("agentRuns", { roomId, agentId: "a", model: "z-ai/glm-5.2", goal: "g", steps: 5, toolCalls: 1, conflictsSurvived: 0, - inputTokens: 1000, outputTokens: 100, costUsd: 0.5, ms: 100, exhausted: false, idempotencyKey: "finished-key", createdAt: t0 + 1000, + inputTokens: 1000, outputTokens: 100, costUsd: 0.5, costKind: "exact", ms: 100, exhausted: false, stopReason: "done", idempotencyKey: "finished-key", createdAt: t0 + 1000, }); }); const swept = await t.mutation(internal.credits.sweepExpiredReservations, { now: t0 + 2 * 60 * 60 * 1000 }); @@ -211,15 +216,41 @@ describe("convex credits — overspend, pause, sweep", () => { expect(rc?.lifetimeSpentCredits).toBe(2); expect(rc?.reservedCredits).toBe(0); expect(rc?.availableCredits).toBe(18); + const ledger = await t.run(async (ctx) => ctx.db.query("creditLedger").withIndex("by_reservation", (q) => q.eq("reservationKey", "finished-key")).collect()); + expect(ledger.find((row) => row.kind === "settle")?.costKind).toBe("exact"); }); - it("COST-AWARE sweep: a crashed run with no recorded cost CAPTURES the hold (never refunds spent money)", async () => { + it("COST-AWARE sweep: an exact-zero finished run refunds the hold instead of capturing it", async () => { + const t = convexTest(schema, modules); + const roomId = await seedRoom(t); + const t0 = 2_500_000; + await t.mutation(internal.credits.grantCredits, { roomId, credits: 20, source: "pilot", now: t0 }); + await t.mutation(internal.credits.reserve, { roomId, mode: "deep", reservationKey: "finished-free-key", now: t0 }); + await t.run(async (ctx: any) => { + await ctx.db.insert("agentRuns", { + roomId, agentId: "a", model: "openrouter/free", goal: "g", steps: 2, toolCalls: 0, conflictsSurvived: 0, + inputTokens: 500, outputTokens: 50, costUsd: 0, costKind: "exact", ms: 80, exhausted: false, stopReason: "done", idempotencyKey: "finished-free-key", createdAt: t0 + 1000, + }); + }); + + const swept = await t.mutation(internal.credits.sweepExpiredReservations, { now: t0 + 2 * 60 * 60 * 1000 }); + expect(swept).toMatchObject({ swept: 1, captured: 0 }); + const rc = await readRoomCredits(t, roomId); + expect(rc).toMatchObject({ availableCredits: 20, reservedCredits: 0, lifetimeSpentCredits: 0 }); + const ledger = await t.run(async (ctx) => ctx.db.query("creditLedger").withIndex("by_reservation", (q) => q.eq("reservationKey", "finished-free-key")).collect()); + const settlement = ledger.find((row) => row.kind === "settle"); + expect(settlement).toMatchObject({ costKind: "exact", reason: "swept_settled" }); + expect(Math.abs(settlement?.usd ?? NaN)).toBe(0); + expect(Math.abs(settlement?.credits ?? NaN)).toBe(0); + }); + + it("COST-AWARE sweep: a truly abandoned run with no terminal marker CAPTURES the hold", async () => { const t = convexTest(schema, modules); const roomId = await seedRoom(t); const t0 = 3_000_000; await t.mutation(internal.credits.grantCredits, { roomId, credits: 20, source: "pilot", now: t0 }); await t.mutation(internal.credits.reserve, { roomId, mode: "deep", reservationKey: "crashed-key", now: t0 }); - // A run was claimed (row exists) but crashed mid-LLM: costUsd never recorded (still 0). + // A run was claimed (row exists) but crashed mid-LLM: no stopReason was persisted by finish. await t.run(async (ctx: any) => { await ctx.db.insert("agentRuns", { roomId, agentId: "a", model: "z-ai/glm-5.2", goal: "g", steps: 0, toolCalls: 0, conflictsSurvived: 0, @@ -233,6 +264,8 @@ describe("convex credits — overspend, pause, sweep", () => { expect(rc?.lifetimeSpentCredits).toBe(DEEP_HOLD); expect(rc?.reservedCredits).toBe(0); expect(rc?.availableCredits).toBe(20 - DEEP_HOLD); + const ledger = await t.run(async (ctx) => ctx.db.query("creditLedger").withIndex("by_reservation", (q) => q.eq("reservationKey", "crashed-key")).collect()); + expect(ledger.find((row) => row.kind === "settle")).toMatchObject({ costKind: "estimated", reason: "swept_captured" }); }); it("admin snapshot rolls up enrolled rooms and spend", async () => { diff --git a/tests/convexModelFreeAutoMode.test.ts b/tests/convexModelFreeAutoMode.test.ts index 3174263a..ba875a94 100644 --- a/tests/convexModelFreeAutoMode.test.ts +++ b/tests/convexModelFreeAutoMode.test.ts @@ -1,5 +1,8 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { z } from "zod"; +import { RoomEngine } from "../src/engine/roomEngine"; +import { buildDemoRoom } from "../src/engine/demoRoom"; +import { AgentRunError, InMemoryRoomTools, ROOM_TOOLS, runAgent } from "../src/nodeagent"; import type { AgentTool } from "../src/nodeagent/core/types"; import { convexModel } from "../src/nodeagent/models/convexModel"; import { openRouterFreeRouteHealthSnapshot, resetOpenRouterFreeRouteHealth } from "../src/nodeagent/models/openRouterFreeModels"; @@ -26,6 +29,12 @@ const freeModels = [ }, ]; +function runtimeTools() { + const engine = new RoomEngine(); + const demo = buildDemoRoom(engine); + return new InMemoryRoomTools(engine, demo.roomId, demo.sheetId, demo.agents.room, demo.sessions.room); +} + describe("Convex free-auto model routing", () => { beforeEach(() => { process.env.OPENROUTER_API_KEY = "test-openrouter-key"; @@ -45,6 +54,13 @@ describe("Convex free-auto model routing", () => { delete process.env.OPENROUTER_FREE_CANDIDATE_TIMEOUT_MS; delete process.env.OPENROUTER_FREE_REQUEST_TIMEOUT_MS; delete process.env.OPENROUTER_FREE_REQUEST_RESERVE_MS; + delete process.env.NEBIUS_API_KEY; + delete process.env.OPENAI_API_KEY; + delete process.env.ANTHROPIC_API_KEY; + delete process.env.GOOGLE_GENERATIVE_AI_API_KEY; + delete process.env.AGENT_FALLBACK_MODEL; + delete process.env.AGENT_FALLBACK_MODELS; + delete process.env.AGENT_QUALITY_CANDIDATE_TIMEOUT_MS; resetOpenRouterFreeRouteHealth(); vi.useRealTimers(); }); @@ -151,7 +167,7 @@ describe("Convex free-auto model routing", () => { ])); }); - it("rotates prose-only output and cools the route when a tool call is required", async () => { + it("returns a required-tool miss to the runtime instead of preempting protocol recovery", async () => { const attempted: string[] = []; vi.stubGlobal("fetch", vi.fn(async (input: string | URL | Request, init?: RequestInit) => { const url = String(input); @@ -160,24 +176,9 @@ describe("Convex free-auto model routing", () => { } const body = JSON.parse(String(init?.body ?? "{}")) as { model?: string }; attempted.push(String(body.model)); - if (attempted.length === 1) { - return new Response(JSON.stringify({ - choices: [{ message: { content: "I will make that change." } }], - usage: { prompt_tokens: 10, completion_tokens: 5 }, - }), { status: 200, headers: { "Content-Type": "application/json" } }); - } return new Response(JSON.stringify({ - choices: [{ - message: { - content: null, - tool_calls: [{ - id: "call-say-1", - type: "function", - function: { name: "say", arguments: JSON.stringify({ text: "done" }) }, - }], - }, - }], - usage: { prompt_tokens: 12, completion_tokens: 7 }, + choices: [{ message: { content: "I will make that change." } }], + usage: { prompt_tokens: 10, completion_tokens: 5 }, }), { status: 200, headers: { "Content-Type": "application/json" } }); })); @@ -195,30 +196,419 @@ describe("Convex free-auto model routing", () => { toolChoice: "required", }); - expect(attempted).toEqual([ - "nvidia/nemotron-3-super-120b-a12b:free", - "qwen/qwen3-next-80b-a3b-instruct:free", - ]); - expect(route.name).toBe("qwen/qwen3-next-80b-a3b-instruct:free"); - expect(step.toolCalls).toEqual([{ - id: "call-say-1", - tool: "say", - args: { text: "done" }, - }]); + expect(attempted).toEqual(["nvidia/nemotron-3-super-120b-a12b:free"]); + expect(route.name).toBe("nvidia/nemotron-3-super-120b-a12b:free"); + expect(step.text).toBe("I will make that change."); + expect(step.toolCalls).toEqual([]); expect(step.providerRoute).toMatchObject({ - requestedModel: "qwen/qwen3-next-80b-a3b-instruct:free", - resolvedModel: "qwen/qwen3-next-80b-a3b-instruct:free", + requestedModel: "nvidia/nemotron-3-super-120b-a12b:free", + resolvedModel: "nvidia/nemotron-3-super-120b-a12b:free", provider: "openrouter", entrypoint: "free", }); const health = openRouterFreeRouteHealthSnapshot(); expect(health.find((entry) => entry.modelId === "nvidia/nemotron-3-super-120b-a12b:free")).toMatchObject({ - successes: 0, - consecutiveFailures: 1, - }); - expect(health.find((entry) => entry.modelId === "qwen/qwen3-next-80b-a3b-instruct:free")).toMatchObject({ successes: 1, consecutiveFailures: 0, }); }); + + it("keeps a successful concrete fallback sticky and the failed primary cooling down", async () => { + process.env.NEBIUS_API_KEY = "test-nebius-key"; + const attempted: string[] = []; + vi.stubGlobal("fetch", vi.fn(async (_input: string | URL | Request, init?: RequestInit) => { + const body = JSON.parse(String(init?.body ?? "{}")) as { model?: string }; + attempted.push(String(body.model)); + if (attempted.length === 1) { + return new Response(JSON.stringify({ + choices: [{ message: { content: "" } }], + usage: { prompt_tokens: 10, completion_tokens: 0 }, + }), { status: 200, headers: { "Content-Type": "application/json" } }); + } + return new Response(JSON.stringify({ + choices: [{ message: { content: `answer from ${body.model}` } }], + usage: { prompt_tokens: 12, completion_tokens: 6 }, + }), { status: 200, headers: { "Content-Type": "application/json" } }); + })); + const route = convexModel("nebius/zai-org/GLM-5.2", { + entrypoint: "public_ask", + fallbackModelIds: ["z-ai/glm-5.2"], + }); + const first = await route.next({ + system: "Answer.", + messages: [{ role: "user", content: "first turn" }], + tools: [], + }); + const resumed = convexModel("nebius/zai-org/GLM-5.2", { + entrypoint: "public_ask", + fallbackModelIds: ["z-ai/glm-5.2"], + routeState: route.routeState?.(), + }); + const second = await resumed.next({ + system: "Answer.", + messages: [{ role: "user", content: "second turn" }], + tools: [], + }); + + expect(attempted).toEqual(["zai-org/GLM-5.2", "z-ai/glm-5.2", "z-ai/glm-5.2"]); + expect(route.name).toBe("z-ai/glm-5.2"); + expect(resumed.name).toBe("z-ai/glm-5.2"); + expect(resumed.routeState?.()).toMatchObject({ preferredModelId: "z-ai/glm-5.2" }); + expect(first.usage).toMatchObject({ inputTokens: 22, outputTokens: 6, modelCalls: 2 }); + expect(second.usage).toMatchObject({ inputTokens: 12, outputTokens: 6, modelCalls: 1 }); + const quality = (first.providerRoute as { qualityFailover?: { budget: { spentCostUsd: number } } } | undefined)?.qualityFailover; + expect(first.usage?.costUsd).toBe(quality?.budget.spentCostUsd); + expect(quality).toMatchObject({ + status: "succeeded", + selectedRouteId: "z-ai/glm-5.2", + routeAttempts: [ + { routeId: "nebius/zai-org/GLM-5.2", outcome: "quality_failure", reason: "empty_result" }, + { routeId: "z-ai/glm-5.2", outcome: "accepted" }, + ], + }); + }); + + it("stops concrete failover on provider policy rejection", async () => { + process.env.NEBIUS_API_KEY = "test-nebius-key"; + const attempted: string[] = []; + vi.stubGlobal("fetch", vi.fn(async (_input: string | URL | Request, init?: RequestInit) => { + const body = JSON.parse(String(init?.body ?? "{}")) as { model?: string }; + attempted.push(String(body.model)); + return new Response("provider policy blocked", { status: 403 }); + })); + + const route = convexModel("nebius/zai-org/GLM-5.2", { + entrypoint: "public_ask", + fallbackModelIds: ["z-ai/glm-5.2"], + }); + const error = await route.next({ + system: "Answer briefly.", + messages: [{ role: "user", content: "hello" }], + tools: [], + }).then(() => undefined, (failure: unknown) => failure); + + expect(error).toBeInstanceOf(QualityFailoverError); + if (!(error instanceof QualityFailoverError)) throw error; + expect(attempted).toEqual(["zai-org/GLM-5.2"]); + expect(error.receipt).toMatchObject({ + stopReason: "global_provider_failure", + terminalFailure: { providerFailureCategory: "policy", providerFailureScope: "global" }, + routeAttempts: [{ outcome: "provider_failure", decision: "stop" }], + }); + }); + + it("does not issue a hidden blocking request after a failed stream", async () => { + const attempted: string[] = []; + const streamed: string[] = []; + const encoder = new TextEncoder(); + vi.stubGlobal("fetch", vi.fn(async (_input: string | URL | Request, init?: RequestInit) => { + const body = JSON.parse(String(init?.body ?? "{}")) as { model?: string; stream?: boolean }; + attempted.push(String(body.model)); + expect(body.stream).toBe(true); + let delivered = false; + const stream = new ReadableStream({ + pull(controller) { + if (!delivered) { + delivered = true; + controller.enqueue(encoder.encode( + 'data: {"choices":[{"delta":{"content":"partial"}}]}\n\n' + + 'data: {"choices":[],"usage":{"prompt_tokens":5,"completion_tokens":2,"prompt_tokens_details":{"cached_tokens":1}}}\n\n', + )); + return; + } + controller.error(new Error("stream transport failed")); + }, + }); + return new Response(stream, { status: 200, headers: { "Content-Type": "text/event-stream" } }); + })); + + const route = convexModel("z-ai/glm-5.2", { entrypoint: "public_ask" }); + const error = await route.next({ + system: "Answer briefly.", + messages: [{ role: "user", content: "hello" }], + tools: [], + onTextDelta: (text) => { + streamed.push(text); + }, + }).then(() => undefined, (failure: unknown) => failure); + + expect(error).toBeInstanceOf(QualityFailoverError); + expect(attempted).toEqual(["z-ai/glm-5.2"]); + expect(streamed).toEqual([]); + expect((error as QualityFailoverError).usage).toMatchObject({ + inputTokens: 5, + outputTokens: 2, + cachedInputTokens: 1, + modelCalls: 1, + costKind: "exact", + }); + expect((error as QualityFailoverError).usage?.costUsd).toBeCloseTo(0.0000081, 10); + }); + + it("propagates failed failover usage and cached-token cost into AgentRunError", async () => { + process.env.NEBIUS_API_KEY = "test-nebius-key"; + const attempted: string[] = []; + vi.stubGlobal("fetch", vi.fn(async (_input: string | URL | Request, init?: RequestInit) => { + const body = JSON.parse(String(init?.body ?? "{}")) as { model?: string }; + attempted.push(String(body.model)); + const isPrimary = attempted.length === 1; + return new Response(JSON.stringify({ + choices: [{ message: { content: "" } }], + usage: { + prompt_tokens: isPrimary ? 10 : 12, + completion_tokens: 0, + prompt_tokens_details: { cached_tokens: isPrimary ? 4 : 2 }, + }, + }), { status: 200, headers: { "Content-Type": "application/json" } }); + })); + const route = convexModel("nebius/zai-org/GLM-5.2", { + entrypoint: "public_ask", + fallbackModelIds: ["z-ai/glm-5.2"], + }); + + const thrown = await runAgent({ + rt: runtimeTools(), + goal: "answer the workbook question", + model: route, + tools: [], + maxSteps: 2, + }).then(() => undefined, (error: unknown) => error); + + expect(thrown).toBeInstanceOf(AgentRunError); + const error = thrown as AgentRunError; + expect(error.cause).toBeInstanceOf(QualityFailoverError); + expect(attempted).toEqual(["zai-org/GLM-5.2", "z-ai/glm-5.2"]); + expect(error.partial.usage).toMatchObject({ + inputTokens: 22, + outputTokens: 0, + cachedInputTokens: 6, + modelCalls: 2, + }); + expect(error.partial.usage.costUsd).toBeCloseTo(0.00001806, 10); + expect((error.cause as QualityFailoverError).usage).toEqual(error.partial.usage); + }); + + it("lets runAgent terminate required-tool misses as protocol_stall after four calls", async () => { + const attempted: string[] = []; + vi.stubGlobal("fetch", vi.fn(async (_input: string | URL | Request, init?: RequestInit) => { + const body = JSON.parse(String(init?.body ?? "{}")) as { model?: string }; + attempted.push(String(body.model)); + return new Response(JSON.stringify({ + choices: [{ message: { content: "I will update it." } }], + usage: { prompt_tokens: 10, completion_tokens: 5 }, + }), { status: 200, headers: { "Content-Type": "application/json" } }); + })); + + const result = await runAgent({ + rt: runtimeTools(), + goal: "write 42 into the workbook cells", + model: convexModel("z-ai/glm-5.2", { entrypoint: "public_ask" }), + tools: ROOM_TOOLS, + maxSteps: 8, + }); + + expect(attempted).toEqual(Array(4).fill("z-ai/glm-5.2")); + expect(result.stopReason).toBe("step_budget"); + expect(result.handoff).toMatchObject({ terminalReason: "protocol_stall", remainingToolCalls: [] }); + expect(result.usage).toMatchObject({ modelCalls: 4, inputTokens: 40, outputTokens: 20 }); + }); + + it("stops a same-provider auth failure but rotates to an authorized cross-provider route", async () => { + process.env.NEBIUS_API_KEY = "test-nebius-key"; + const sameProviderAttempts: string[] = []; + vi.stubGlobal("fetch", vi.fn(async (_input: string | URL | Request, init?: RequestInit) => { + const body = JSON.parse(String(init?.body ?? "{}")) as { model?: string }; + sameProviderAttempts.push(String(body.model)); + return new Response("unauthorized", { status: 401 }); + })); + const sameProvider = convexModel("z-ai/glm-5.2", { + entrypoint: "public_ask", + fallbackModelIds: ["z-ai/glm-4.7"], + }); + const sameProviderError = await sameProvider.next({ + system: "Answer.", + messages: [{ role: "user", content: "hello" }], + tools: [], + }).then(() => undefined, (error: unknown) => error); + expect(sameProviderError).toBeInstanceOf(QualityFailoverError); + expect(sameProviderAttempts).toEqual(["z-ai/glm-5.2"]); + expect((sameProviderError as QualityFailoverError).receipt.stopReason).toBe("global_provider_failure"); + + const crossProviderAttempts: string[] = []; + vi.stubGlobal("fetch", vi.fn(async (_input: string | URL | Request, init?: RequestInit) => { + const body = JSON.parse(String(init?.body ?? "{}")) as { model?: string }; + crossProviderAttempts.push(String(body.model)); + if (crossProviderAttempts.length === 1) return new Response("unauthorized", { status: 401 }); + return new Response(JSON.stringify({ + choices: [{ message: { content: "fallback answer" } }], + usage: { prompt_tokens: 5, completion_tokens: 2 }, + }), { status: 200, headers: { "Content-Type": "application/json" } }); + })); + const crossProvider = convexModel("nebius/zai-org/GLM-5.2", { + entrypoint: "public_ask", + fallbackModelIds: ["z-ai/glm-5.2"], + }); + const result = await crossProvider.next({ + system: "Answer.", + messages: [{ role: "user", content: "hello" }], + tools: [], + }); + expect(crossProviderAttempts).toEqual(["zai-org/GLM-5.2", "z-ai/glm-5.2"]); + expect(result.text).toBe("fallback answer"); + }); + + it("does not label unknown catalog pricing as exact cost", async () => { + vi.stubGlobal("fetch", vi.fn(async () => new Response(JSON.stringify({ + choices: [{ message: { content: "answer" } }], + usage: { prompt_tokens: 5, completion_tokens: 2 }, + }), { status: 200, headers: { "Content-Type": "application/json" } }))); + const step = await convexModel("vendor/unknown-paid-model", { entrypoint: "public_ask" }).next({ + system: "Answer.", + messages: [{ role: "user", content: "hello" }], + tools: [], + }); + expect(step.usage).toMatchObject({ inputTokens: 5, outputTokens: 2, modelCalls: 1 }); + expect(step.usage).not.toHaveProperty("costUsd"); + + const result = await runAgent({ + rt: runtimeTools(), + goal: "answer the question", + model: convexModel("vendor/unknown-paid-model", { entrypoint: "public_ask" }), + tools: [], + maxSteps: 1, + priceStep: () => 0.01, + }); + expect(result.usage).toMatchObject({ modelCalls: 1, costKind: "estimated" }); + expect(result.usage.costUsd).toBeGreaterThan(0); + }); + + it("counts no provider call when a direct provider key is missing", async () => { + const fetchSpy = vi.fn(); + vi.stubGlobal("fetch", fetchSpy); + + const error = await convexModel("gpt-5.4", { entrypoint: "public_ask" }).next({ + system: "Answer.", + messages: [{ role: "user", content: "hello" }], + tools: [], + }).then(() => undefined, (failure: unknown) => failure); + + expect(error).toBeInstanceOf(QualityFailoverError); + expect(fetchSpy).not.toHaveBeenCalled(); + expect((error as QualityFailoverError).usage).toMatchObject({ + inputTokens: 0, + outputTokens: 0, + modelCalls: 0, + costUsd: 0, + costKind: "exact", + }); + }); + + it("labels a timed-out provider request estimated and checkpoints its cooldown", async () => { + vi.useFakeTimers(); + process.env.NEBIUS_API_KEY = "test-nebius-key"; + process.env.AGENT_QUALITY_CANDIDATE_TIMEOUT_MS = "10000"; + vi.stubGlobal("fetch", vi.fn(async (_input: string | URL | Request, init?: RequestInit) => new Promise((_resolve, reject) => { + init?.signal?.addEventListener("abort", () => { + setTimeout(() => reject(new Error("provider fetch aborted after timeout")), 5); + }, { once: true }); + }))); + + const route = convexModel("nebius/zai-org/GLM-5.2", { entrypoint: "public_ask" }); + const pending = route.next({ + system: "Answer.", + messages: [{ role: "user", content: "hello" }], + tools: [], + }).then(() => undefined, (failure: unknown) => failure); + + await vi.advanceTimersByTimeAsync(10_005); + const error = await pending; + + expect(error).toBeInstanceOf(QualityFailoverError); + expect((error as QualityFailoverError).usage).toMatchObject({ + inputTokens: 0, + outputTokens: 0, + modelCalls: 1, + costUsd: 0, + costKind: "estimated", + }); + expect(route.routeState?.()).toMatchObject({ + cooldownUntil: { "nebius/zai-org/GLM-5.2": expect.any(Number) }, + }); + }); + + it("accounts for Anthropic cache reads and labels unknown cache-write pricing estimated", async () => { + process.env.ANTHROPIC_API_KEY = "test-anthropic-key"; + let call = 0; + vi.stubGlobal("fetch", vi.fn(async () => { + call += 1; + return new Response(JSON.stringify({ + content: [{ type: "text", text: "answer" }], + usage: { + input_tokens: 20, + cache_read_input_tokens: 80, + cache_creation_input_tokens: call === 1 ? 0 : 10, + output_tokens: 10, + }, + }), { status: 200, headers: { "Content-Type": "application/json" } }); + })); + + const exact = await convexModel("claude-sonnet-4.6", { entrypoint: "public_ask" }).next({ + system: "Answer.", + messages: [{ role: "user", content: "hello" }], + tools: [], + }); + expect(exact.usage).toMatchObject({ + inputTokens: 100, + outputTokens: 10, + cachedInputTokens: 80, + cacheCreationInputTokens: 0, + modelCalls: 1, + costKind: "exact", + }); + expect(exact.usage?.costUsd).toBeCloseTo(0.000234, 10); + + const estimated = await runAgent({ + rt: runtimeTools(), + goal: "answer the question", + model: convexModel("claude-sonnet-4.6", { entrypoint: "public_ask" }), + tools: [], + maxSteps: 1, + }); + expect(estimated.usage).toMatchObject({ + inputTokens: 110, + outputTokens: 10, + cachedInputTokens: 80, + cacheCreationInputTokens: 10, + modelCalls: 1, + costKind: "estimated", + }); + }); + + it("does not apply deployment fallback env without an artifact-authorized caller list", async () => { + process.env.NEBIUS_API_KEY = "test-nebius-key"; + process.env.AGENT_FALLBACK_MODELS = "z-ai/glm-5.2"; + const attempted: string[] = []; + vi.stubGlobal("fetch", vi.fn(async (_input: string | URL | Request, init?: RequestInit) => { + const body = JSON.parse(String(init?.body ?? "{}")) as { model?: string }; + attempted.push(String(body.model)); + return new Response("provider unavailable", { status: 503 }); + })); + const sayTool: AgentTool = { + name: "say", + description: "Return a message.", + schema: z.object({ text: z.string() }), + execute: async () => ({ ok: true }), + }; + + const route = convexModel("nebius/zai-org/GLM-5.2", { entrypoint: "public_ask" }); + const error = await route.next({ + system: "Call the tool.", + messages: [{ role: "user", content: "finish" }], + tools: [sayTool], + toolChoice: "required", + }).then(() => undefined, (failure: unknown) => failure); + + expect(error).toBeInstanceOf(QualityFailoverError); + expect(attempted).toEqual(["zai-org/GLM-5.2"]); + }); }); diff --git a/tests/gatewayAndJournal.test.ts b/tests/gatewayAndJournal.test.ts index 102bef5b..0c478bfb 100644 --- a/tests/gatewayAndJournal.test.ts +++ b/tests/gatewayAndJournal.test.ts @@ -150,6 +150,62 @@ describe("exactly-once journal (no double-bill on slice retry)", () => { const second = await run(new RoomEngine()); expect(second.r.stopReason).toBe("done"); expect(modelCalls).toBe(0); // ← exactly-once: NO re-call, NO re-bill + expect(second.r.usage.modelCalls).toBe(first.r.usage.modelCalls); // original usage is rematerialized expect(String(second.eng.getArtifact(second.d.sheetId)!.elements[CELL]?.value)).toBe(VAL); // work still completed via replay }); + + it("rematerializes journaled usage after a crash between journal commit and slice accounting", async () => { + const eng = new RoomEngine(); + const d = buildDemoRoom(eng); + const rt = new InMemoryRoomTools(eng, d.roomId, d.sheetId, d.agents.room, d.sessions.room); + let stored: Awaited> | undefined; + let crashAfterCommit = true; + let providerRequests = 0; + const journal = { + get: () => stored, + record: (_step: number, result: Awaited>) => { + stored = result; + if (crashAfterCommit) throw new Error("simulated_crash_after_journal_commit"); + }, + }; + const model: AgentModel = { + name: "journal-accounting-model", + async next() { + providerRequests += 1; + return { + text: "done", + toolCalls: [], + done: true, + usage: { + inputTokens: 17, + outputTokens: 5, + cachedInputTokens: 3, + cacheCreationInputTokens: 2, + modelCalls: 1, + costUsd: 0.004, + costKind: "exact", + }, + }; + }, + }; + + await expect(runAgent({ rt, goal: "Summarize the workbook.", model, tools: [], maxSteps: 2, journal })) + .rejects.toThrow("simulated_crash_after_journal_commit"); + expect(providerRequests).toBe(1); + + crashAfterCommit = false; + providerRequests = 0; + const replay = await runAgent({ rt, goal: "Summarize the workbook.", model, tools: [], maxSteps: 2, journal }); + + expect(providerRequests).toBe(0); + expect(replay.usage).toMatchObject({ + inputTokens: 17, + outputTokens: 5, + cachedInputTokens: 3, + cacheCreationInputTokens: 2, + modelCalls: 1, + costUsd: 0.004, + costKind: "exact", + }); + }); }); diff --git a/tests/idempotencyRuntime.test.ts b/tests/idempotencyRuntime.test.ts index 943f017c..6f561580 100644 --- a/tests/idempotencyRuntime.test.ts +++ b/tests/idempotencyRuntime.test.ts @@ -34,10 +34,32 @@ test("RUNTIME: concurrent double-submit dedupes to the in-flight run; exactly on expect(reuse?.stopReason).toBeUndefined(); // it is in flight // Run #1 finishes by PATCHING the claimed row (not a 2nd insert) → still exactly one run for this key. - await t.mutation(internal.agentRuns.finish, { runId: runId1, model: "gpt-5.4-mini", steps: 4, toolCalls: 6, conflictsSurvived: 1, inputTokens: 200, outputTokens: 80, costUsd: 0.0042, ms: 1800, exhausted: false, stopReason: "done" }); + await t.mutation(internal.agentRuns.finish, { + runId: runId1, + model: "gpt-5.4-mini", + steps: 4, + modelCalls: 2, + toolCalls: 6, + conflictsSurvived: 1, + inputTokens: 200, + outputTokens: 80, + cachedInputTokens: 120, + cacheCreationInputTokens: 15, + costUsd: 0.0042, + costKind: "estimated", + ms: 1800, + exhausted: false, + stopReason: "done", + }); const after = await t.query(internal.agentRuns.byKey, { idempotencyKey: key }); expect(after).toHaveLength(1); // ONE row total — no concurrent duplicate ran expect(after[0].stopReason).toBe("done"); + expect(after[0]).toMatchObject({ + modelCalls: 2, + cachedInputTokens: 120, + cacheCreationInputTokens: 15, + costKind: "estimated", + }); // A rapid re-click within the recency window still dedupes (no double-bill); a different goal does NOT. expect(findReusableRun(mapRows(after), key, { now: Date.now() })?.runId).toBe(String(runId1)); @@ -61,3 +83,38 @@ test("RUNTIME (atomic, race-safe): claimOrReuse — first inserts, second reuses expect(String(second.runId)).toBe(String(first.runId)); expect(await t.query(internal.agentRuns.byKey, { idempotencyKey: key })).toHaveLength(1); // exactly one row }); + +test("RUNTIME: agentRuns.record preserves explicit zero calls and cache accounting", async () => { + const t = convexTest(schema, modules); + const roomId = await t.run((ctx) => + ctx.db.insert("rooms", { code: "TEST03", title: "Accounting", hostId: "u1", autoAllow: true, status: "live" as const, createdAt: Date.now() })); + + const runId = await t.mutation(internal.agentRuns.record, { + roomId, + agentId: "agent_pub", + model: "provider-unavailable", + goal: "Record a preflight failure", + steps: 0, + modelCalls: 0, + toolCalls: 0, + conflictsSurvived: 0, + inputTokens: 0, + outputTokens: 0, + cachedInputTokens: 0, + cacheCreationInputTokens: 0, + costUsd: 0, + costKind: "exact", + ms: 12, + exhausted: false, + stopReason: "provider_unavailable", + }); + const row = await t.run((ctx) => ctx.db.get(runId)); + + expect(row).toMatchObject({ + modelCalls: 0, + toolCalls: 0, + cachedInputTokens: 0, + cacheCreationInputTokens: 0, + costKind: "exact", + }); +}); diff --git a/tests/modelAdapterFreeAutoMode.test.ts b/tests/modelAdapterFreeAutoMode.test.ts index 2ef01ad8..24320d79 100644 --- a/tests/modelAdapterFreeAutoMode.test.ts +++ b/tests/modelAdapterFreeAutoMode.test.ts @@ -115,7 +115,7 @@ describe("model adapter free-auto modes", () => { await vi.advanceTimersByTimeAsync(0); expect(generateTextMock).toHaveBeenCalledTimes(2); - await vi.advanceTimersByTimeAsync(5_000); + await vi.advanceTimersByTimeAsync(5_250); const result = await settled; if (!result.ok) throw result.error; const { step } = result; @@ -161,7 +161,7 @@ describe("model adapter free-auto modes", () => { await vi.advanceTimersByTimeAsync(0); expect(generateTextMock).toHaveBeenCalledTimes(1); - await vi.advanceTimersByTimeAsync(5_000); + await vi.advanceTimersByTimeAsync(5_250); const error = await settled; expect(error).toBeInstanceOf(QualityFailoverError); diff --git a/tests/phaseModel.test.ts b/tests/phaseModel.test.ts index c0cedf65..3d2f4fa0 100644 --- a/tests/phaseModel.test.ts +++ b/tests/phaseModel.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect, afterEach } from "vitest"; -import { modelForFramePhase, ORCHESTRATOR_PHASES } from "../src/nodeagent/models/phaseModel"; +import { authorizedModelForFramePhase, modelForFramePhase, ORCHESTRATOR_PHASES } from "../src/nodeagent/models/phaseModel"; describe("modelForFramePhase", () => { const originalEnv = { ...process.env }; @@ -68,6 +68,14 @@ describe("modelForFramePhase", () => { expect(modelForFramePhase("execute", "fallback")).toBe("nebius/MiniMaxAI/MiniMax-M2.5"); }); + it("falls back when an artifact-aware egress gate rejects the phase override", () => { + const env = { AGENT_ORCHESTRATOR_MODEL: "openrouter/free-auto" }; + expect(authorizedModelForFramePhase("plan", "anthropic/claude-sonnet-4", () => false, env)) + .toBe("anthropic/claude-sonnet-4"); + expect(authorizedModelForFramePhase("plan", "anthropic/claude-sonnet-4", () => true, env)) + .toBe("openrouter/free-auto"); + }); + it("ORCHESTRATOR_PHASES contains the expected set", () => { expect(ORCHESTRATOR_PHASES.has("intake")).toBe(true); expect(ORCHESTRATOR_PHASES.has("plan")).toBe(true); diff --git a/tests/proofloopSupervisor.test.ts b/tests/proofloopSupervisor.test.ts index 5019dbfa..34c5e9ca 100644 --- a/tests/proofloopSupervisor.test.ts +++ b/tests/proofloopSupervisor.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; import { PROOFLOOP_NO_PROGRESS_AFTER_REPAIR, + PROOFLOOP_REPEATED_WORKFLOW_BLOCK, PROOFLOOP_VERIFIER_REPAIR_PREFIX, appendProofloopRepairMessage, proofloopSupervisorDecision, @@ -57,6 +58,74 @@ describe("ProofLoop benchmark supervisor", () => { expect(decision.kind).toBe("none"); }); + it("issues one bounded repair after repeated verified-workbook blocks", () => { + const decision = proofloopSupervisorDecision({ + runtimeProfile: "benchmark_completion", + goal: WRITE_GOAL, + attempt: 1, + maxAttempts: 12, + result: result({ + stopReason: "step_budget", + trace: Array.from({ length: 4 }, (_, step) => ({ + step, + tool: "write_locked_cells", + args: {}, + result: { ok: false, error: "tool_blocked", reason: "verified_workbook_workflow:write_plan_mismatch: blocked" }, + ms: 1, + })), + }), + }); + + expect(decision).toMatchObject({ kind: "repair" }); + if (decision.kind === "repair") { + expect(decision.prompt).toContain("one unchanged approved operation"); + expect(decision.prompt).not.toContain("approvedWrite"); + } + }); + + it("fails repeated workflow blocks after the bounded repair is exhausted", () => { + const decision = proofloopSupervisorDecision({ + runtimeProfile: "benchmark_completion", + goal: WRITE_GOAL, + attempt: 2, + maxAttempts: 12, + result: result({ + stopReason: "step_budget", + messages: [ + { role: "user", content: `${PROOFLOOP_VERIFIER_REPAIR_PREFIX} bind the approved plan` }, + ...Array.from({ length: 4 }, (_, index) => ({ + role: "tool" as const, + toolCallId: `blocked-${index}`, + toolName: "write_locked_cells", + content: JSON.stringify({ ok: false, error: "tool_blocked", reason: "verified_workbook_workflow:write_plan_mismatch: blocked" }), + })), + ], + }), + }); + + expect(decision).toMatchObject({ kind: "terminal_failure", error: PROOFLOOP_NO_PROGRESS_AFTER_REPAIR }); + }); + + it("does not treat blocked write calls as successful receipts", () => { + const decision = proofloopSupervisorDecision({ + runtimeProfile: "benchmark_completion", + goal: WRITE_GOAL, + attempt: 2, + maxAttempts: 12, + result: result({ + trace: Array.from({ length: 4 }, (_, step) => ({ + step, + tool: "write_locked_cells", + args: {}, + result: { ok: false, error: "tool_blocked", reason: "verified_workbook_workflow:preflight_required: blocked" }, + ms: 1, + })), + }), + }); + + expect(decision).toMatchObject({ kind: "terminal_failure", error: PROOFLOOP_REPEATED_WORKFLOW_BLOCK }); + }); + it("ignores non-benchmark and read-only goals", () => { expect(proofloopSupervisorDecision({ runtimeProfile: undefined, diff --git a/tests/qualityFailover.test.ts b/tests/qualityFailover.test.ts index f20bde51..7519c97d 100644 --- a/tests/qualityFailover.test.ts +++ b/tests/qualityFailover.test.ts @@ -356,7 +356,7 @@ describe("quality-aware bounded failover", () => { now: () => 8_000, }); - await vi.advanceTimersByTimeAsync(25); + await vi.advanceTimersByTimeAsync(300); const result = await pending; expect(result.ok).toBe(true); @@ -378,6 +378,37 @@ describe("quality-aware bounded failover", () => { ]); }); + it("captures usage that an aborted provider attaches while settling the timeout", async () => { + vi.useFakeTimers(); + const onRouteAttempt = vi.fn(); + const pending = runQualityFailover({ + candidates: [candidate("partial-timeout")], + budget: { maxAttempts: 1 }, + attemptTimeoutMs: 25, + execute: async (_route, context) => new Promise((_resolve, reject) => { + context.signal.addEventListener("abort", () => { + setTimeout(() => reject(Object.assign(new Error("aborted after partial usage"), { + usage: { inputTokens: 19, outputTokens: 4, modelCalls: 1, costUsd: 0.125, costKind: "exact" }, + })), 5); + }, { once: true }); + }), + measureCostUsd: ({ error }) => Number((error as { usage?: { costUsd?: number } } | undefined)?.usage?.costUsd ?? 0), + onRouteAttempt, + now: () => 9_000, + }); + + await vi.advanceTimersByTimeAsync(30); + const result = await pending; + + expect(result.ok).toBe(false); + expect(result.receipt).toMatchObject({ + stopReason: "candidates_exhausted", + budget: { spentCostUsd: 0.125 }, + routeAttempts: [{ routeId: "partial-timeout", reason: "candidate_timeout", costUsd: 0.125 }], + }); + expect(onRouteAttempt).toHaveBeenCalledTimes(1); + }); + it("uses the default non-empty quality floor without a provider", () => { expect(assessNonEmptyResult(undefined)).toEqual({ ok: false, reason: "empty_result" }); expect(assessNonEmptyResult(" ")).toEqual({ ok: false, reason: "empty_result" }); diff --git a/tests/roomShellTelemetry.test.ts b/tests/roomShellTelemetry.test.ts new file mode 100644 index 00000000..c22db63f --- /dev/null +++ b/tests/roomShellTelemetry.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from "vitest"; +import { projectAgentJobAttemptTelemetry, projectAgentRunTelemetry } from "../src/app/store"; +import { formatAgentCost } from "../src/ui/RoomShell"; + +const persistedRun = { + model: "provider/known-model", + steps: 2, + toolCalls: 1, + inputTokens: 120, + outputTokens: 30, + costUsd: 0.125, + ms: 400, +}; + +describe("RoomShell cost telemetry", () => { + it("preserves estimated run pricing through the store projection and labels it approximately", () => { + const run = projectAgentRunTelemetry({ ...persistedRun, costKind: "estimated" }); + + expect(run.costKind).toBe("estimated"); + expect(formatAgentCost(run.costUsd, run.costKind)).toBe("≈$0.125"); + }); + + it("keeps exact run pricing exact-looking", () => { + const run = projectAgentRunTelemetry({ ...persistedRun, costKind: "exact" }); + + expect(run.costKind).toBe("exact"); + expect(formatAgentCost(run.costUsd, run.costKind)).toBe("$0.125"); + }); + + it("treats legacy telemetry without a cost kind as estimated", () => { + const run = projectAgentRunTelemetry(persistedRun); + const attempt = projectAgentJobAttemptTelemetry({ + attempt: 1, + status: "completed", + resolvedModel: persistedRun.model, + stopReason: "done", + ms: persistedRun.ms, + inputTokens: persistedRun.inputTokens, + outputTokens: persistedRun.outputTokens, + costUsd: persistedRun.costUsd, + }); + + expect(run.costKind).toBe("estimated"); + expect(attempt.costKind).toBe("estimated"); + expect(formatAgentCost(run.costUsd, run.costKind)).toBe("≈$0.125"); + }); +}); diff --git a/tests/workArtifacts.test.ts b/tests/workArtifacts.test.ts index 118e56d6..21b7b535 100644 --- a/tests/workArtifacts.test.ts +++ b/tests/workArtifacts.test.ts @@ -626,7 +626,7 @@ describe("work artifact adapters", () => { roomId: "room-1", messages, traces: [trace], - run: { model: "openrouter/free", steps: 4, toolCalls: 3, inputTokens: 1200, outputTokens: 320, costUsd: 0.012, ms: 1800 }, + run: { model: "openrouter/free", steps: 4, toolCalls: 3, inputTokens: 1200, outputTokens: 320, costUsd: 0.012, costKind: "estimated", ms: 1800 }, job: { id: "job-1", status: "running", @@ -639,7 +639,7 @@ describe("work artifact adapters", () => { modelCallCount: 1, receiptCount: 2, }, - attempts: [{ attempt: 1, status: "running", resolvedModel: "openrouter/free", stopReason: "in_progress", ms: 900, inputTokens: 600, outputTokens: 120, costUsd: 0.004 }], + attempts: [{ attempt: 1, status: "running", resolvedModel: "openrouter/free", stopReason: "in_progress", ms: 900, inputTokens: 600, outputTokens: 120, costUsd: 0.004, costKind: "estimated" }], detail: { operations: [{ sequence: 1, kind: "mutation", name: "patch_bundle_cas", status: "completed" }], streamEvents: [{ sequence: 1, kind: "message_done", status: "completed", createdAt: 20, text: "done" }], diff --git a/tests/workbookAgentTools.test.ts b/tests/workbookAgentTools.test.ts index eca7a002..274c3f9c 100644 --- a/tests/workbookAgentTools.test.ts +++ b/tests/workbookAgentTools.test.ts @@ -51,6 +51,32 @@ describe("NodeAgent workbook planning tools", () => { ])); expect(wrong.repairPrompt).toContain("formula_self_reference"); + const preflight = await tool("verify_workbook").execute({ + instruction, + artifactId: "attendance", + afterWrite: false, + operations: [{ elementId: "F3", formula: 'TEXT(F4,"ddd")', result: "Tue" }], + }, rt) as { + ok: boolean; + status: string; + approvedOperations?: Array<{ elementId: string; baseVersion: number }>; + }; + expect(preflight).toMatchObject({ + ok: true, + status: "passed", + approvedOperations: [{ elementId: "F3", baseVersion: 3 }], + }); + + const stale = await tool("verify_workbook").execute({ + instruction, + artifactId: "attendance", + afterWrite: false, + operations: [{ elementId: "F3", baseVersion: 2, formula: 'TEXT(F4,"ddd")', result: "Tue" }], + }, rt) as { ok: boolean; status: string; approvedOperations?: unknown; repairPrompt?: string }; + expect(stale).toMatchObject({ ok: false, status: "needs_repair" }); + expect(stale).not.toHaveProperty("approvedOperations"); + expect(stale.repairPrompt).toContain("stale_target_version"); + values.set("F3", { value: "Tue", formula: 'TEXT(F4,"ddd")' }); const repaired = await tool("verify_workbook").execute({ instruction, diff --git a/tests/workbookWorkflowHook.test.ts b/tests/workbookWorkflowHook.test.ts index 4448c202..e2652d38 100644 --- a/tests/workbookWorkflowHook.test.ts +++ b/tests/workbookWorkflowHook.test.ts @@ -5,14 +5,56 @@ import { goalRequiresVerifiedWorkbookWorkflow, runAgent, scriptedModel, + PRODUCTION_ROOM_TOOLS, type AgentMessage, type AgentTool, type RoomTools, } from "../src/nodeagent/index"; const OPERATION = { elementId: "B2", formula: "=A2*2", result: 20, numFmt: "0.00" }; +const DEFAULT_TEST_BASE_VERSION = 0; -function workbookTools(writeResult: unknown = { ok: true }): AgentTool[] { +function authoritativeOperations( + operations: Array>, + versions: Record = {}, +): Array> { + return operations.map((operation) => ({ + ...operation, + baseVersion: typeof operation.baseVersion === "number" + ? operation.baseVersion + : versions[String(operation.elementId)] ?? DEFAULT_TEST_BASE_VERSION, + })); +} + +function boundOperations( + operations: Array>, + versions: Record = {}, +): Array> { + return authoritativeOperations(operations, versions).map((operation) => ({ + ...operation, + ...(typeof operation.formula === "string" ? { formula: operation.formula.replace(/^=/, "") } : {}), + })); +} + +function preflightResult( + artifactId: string, + operations: Array>, + versions: Record = {}, +) { + return { + ok: true, + status: "passed", + artifactId, + phase: "preflight", + approvedOperations: authoritativeOperations(operations, versions), + }; +} + +function workbookTools( + writeResult: unknown = { ok: true }, + onWrite?: (args: unknown) => void, + preflightVersions: Record = {}, +): AgentTool[] { return [ { name: "inspect_workbook", @@ -28,6 +70,7 @@ function workbookTools(writeResult: unknown = { ok: true }): AgentTool[] { artifactId: z.string().optional(), operations: z.array(z.object({ elementId: z.string(), + baseVersion: z.number().int().optional(), formula: z.string().optional(), value: z.unknown().optional(), result: z.unknown().optional(), @@ -35,12 +78,18 @@ function workbookTools(writeResult: unknown = { ok: true }): AgentTool[] { })), afterWrite: z.boolean().optional(), }), - execute: async (args: { artifactId?: string; afterWrite?: boolean }) => ({ - ok: true, - status: "passed", - artifactId: args.artifactId ?? "sheet-1", - phase: args.afterWrite === false ? "preflight" : "post_write", - }), + execute: async (args: { + artifactId?: string; + operations: Array>; + afterWrite?: boolean; + }) => args.afterWrite === false + ? preflightResult(args.artifactId ?? "sheet-1", args.operations, preflightVersions) + : { + ok: true, + status: "passed", + artifactId: args.artifactId ?? "sheet-1", + phase: "post_write", + }, }, { name: "write_locked_cells", @@ -49,13 +98,17 @@ function workbookTools(writeResult: unknown = { ok: true }): AgentTool[] { artifactId: z.string().optional(), ops: z.array(z.object({ elementId: z.string(), + baseVersion: z.number().int().optional(), formula: z.string().optional(), value: z.unknown().optional(), result: z.unknown().optional(), numFmt: z.string().optional(), })), }), - execute: async () => writeResult, + execute: async (args: unknown) => { + onWrite?.(args); + return writeResult; + }, }, ]; } @@ -64,11 +117,12 @@ function runWorkbookAgent(args: { goal?: string; model: ReturnType; tools?: AgentTool[]; + rt?: RoomTools; initialMessages?: AgentMessage[]; maxSteps?: number; }) { return runAgent({ - rt: {} as RoomTools, + rt: args.rt ?? ({} as RoomTools), goal: args.goal ?? "Complete this SpreadsheetBench workbook formula task.", model: args.model, tools: args.tools ?? workbookTools(), @@ -132,14 +186,365 @@ describe("verified workbook workflow hook", () => { expect(result.finalText).toBe("Correct plan verified."); }); + it("binds a truncated large write to the complete preflight-approved plan", async () => { + const operations = Array.from({ length: 46 }, (_, index) => { + const row = index + 2; + return { elementId: `B${row}`, formula: `=A${row}*2`, result: row * 2, numFmt: "0.00" }; + }); + const writes: unknown[] = []; + let turn = 0; + const model = scriptedModel(() => { + turn += 1; + if (turn === 1) return { toolCalls: [{ tool: "inspect_workbook", args: { instruction: "fill formulas", artifactId: "sheet-1" } }] }; + if (turn === 2) return { toolCalls: [{ tool: "verify_workbook", args: { instruction: "fill formulas", artifactId: "sheet-1", operations, afterWrite: false } }] }; + if (turn === 3) return { + toolCalls: [{ + tool: "write_locked_cells", + args: { + artifactId: "sheet-1\nwrite_locked_cellsartifactId", + ops: [operations[17]], + }, + }], + }; + if (turn === 4) return { toolCalls: [{ tool: "verify_workbook", args: { instruction: "fill formulas", artifactId: "sheet-1", operations, afterWrite: true } }] }; + return { say: "Large plan committed and verified.", done: true }; + }); + + const result = await runWorkbookAgent({ + model, + tools: workbookTools({ ok: true }, (args) => writes.push(args)), + }); + + expect(result.stopReason).toBe("done"); + expect(writes).toHaveLength(1); + expect(writes[0]).toEqual({ artifactId: "sheet-1", ops: boundOperations(operations) }); + expect((writes[0] as { ops: unknown[] }).ops).toHaveLength(46); + expect(result.trace.find((event) => event.tool === "write_locked_cells")?.args).toEqual(writes[0]); + }); + + it("requires the preflight-approved baseVersion instead of accepting a silent rebase", async () => { + const approvedOperations = [ + { ...OPERATION, baseVersion: 7 }, + { elementId: "C2", formula: "=B2+1", result: 21, numFmt: "0.00", baseVersion: 11 }, + ]; + const writes: unknown[] = []; + let turn = 0; + const model = scriptedModel(() => { + turn += 1; + if (turn === 1) return { toolCalls: [{ tool: "inspect_workbook", args: { instruction: "repair formulas", artifactId: "sheet-1" } }] }; + if (turn === 2) return { toolCalls: [{ tool: "verify_workbook", args: { instruction: "repair formulas", artifactId: "sheet-1", operations: approvedOperations, afterWrite: false } }] }; + if (turn === 3) return { toolCalls: [{ tool: "write_locked_cells", args: { artifactId: "sheet-1", ops: [{ ...approvedOperations[0], baseVersion: 8 }] } }] }; + if (turn === 4) return { toolCalls: [{ tool: "write_locked_cells", args: { artifactId: "sheet-1", ops: [approvedOperations[1]] } }] }; + if (turn === 5) return { toolCalls: [{ tool: "verify_workbook", args: { instruction: "repair formulas", artifactId: "sheet-1", operations: approvedOperations, afterWrite: true } }] }; + return { say: "Approved CAS plan verified.", done: true }; + }); + + const result = await runWorkbookAgent({ + model, + tools: workbookTools({ ok: true }, (args) => writes.push(args)), + }); + const mismatch = result.trace.find((event) => + event.tool === "write_locked_cells" + && (event.result as { reason?: string }).reason?.includes("write_plan_mismatch")); + + expect(mismatch).toBeDefined(); + expect(writes).toEqual([{ artifactId: "sheet-1", ops: boundOperations(approvedOperations) }]); + expect(result.stopReason).toBe("done"); + }); + + it("binds model-omitted authoritative baseVersions so a concurrent human edit produces a CAS conflict", async () => { + const artifactId = "sheet-1"; + const targetIds = ["B2", "C2"]; + const cells = new Map([ + [targetIds[0], { value: 10 as unknown, version: 7 }], + [targetIds[1], { value: 20 as unknown, version: 11 }], + ]); + const before = Object.fromEntries([...cells].map(([elementId, cell]) => [elementId, { ...cell }])); + const editCalls: Array<{ elementId: string; value: unknown; baseVersion: number }> = []; + const rt = { + readRange: async (elementIds: string[]) => elementIds.map((elementId) => { + const cell = cells.get(elementId)!; + return { id: elementId, value: cell.value, version: cell.version, locked: null }; + }), + proposeLock: async () => ({ ok: true as const, lockId: "lock-1" }), + releaseLock: async () => ({ ok: true, merged: [] }), + editCell: async (elementId: string, value: unknown, baseVersion: number) => { + const cell = cells.get(elementId)!; + if (cell.version !== baseVersion) return { ok: false as const, conflict: true as const, expected: baseVersion, actual: cell.version }; + cell.value = value; + cell.version += 1; + editCalls.push({ elementId, value, baseVersion }); + return { ok: true as const, version: cell.version }; + }, + } as unknown as RoomTools; + const operations = [ + { elementId: targetIds[0], value: "+24%" }, + { elementId: targetIds[1], value: "+27.5%" }, + ]; + const productionWrite = PRODUCTION_ROOM_TOOLS.find((tool) => tool.name === "write_locked_cells")!; + const authoritativeVersions = { + [targetIds[0]]: before[targetIds[0]].version, + [targetIds[1]]: before[targetIds[1]].version, + }; + const tools = [ + ...workbookTools({ ok: true }, undefined, authoritativeVersions).filter((tool) => tool.name !== "write_locked_cells"), + productionWrite, + ]; + let humanEditApplied = false; + let turn = 0; + const model = scriptedModel(() => { + turn += 1; + if (turn === 1) return { toolCalls: [{ tool: "inspect_workbook", args: { instruction: "fill variances", artifactId } }] }; + if (turn === 2) return { toolCalls: [{ tool: "verify_workbook", args: { instruction: "fill variances", artifactId, operations, afterWrite: false } }] }; + if (turn === 3) { + const humanTarget = cells.get(targetIds[1])!; + humanTarget.value = "+19% human"; + humanTarget.version += 1; + humanEditApplied = true; + return { + toolCalls: [{ + tool: "write_locked_cells", + args: { + artifactId: `${artifactId}\nwrite_locked_cellsartifactId`, + ops: [operations[0]], + }, + }], + }; + } + return { say: "Write encountered a conflict.", done: true }; + }); + + const result = await runWorkbookAgent({ model, tools, rt, maxSteps: 4 }); + const write = result.trace.find((event) => event.tool === "write_locked_cells"); + + expect(humanEditApplied).toBe(true); + expect(write?.args).toEqual({ artifactId, ops: boundOperations(operations, authoritativeVersions) }); + expect(write?.result).toMatchObject({ + ok: false, + conflict: true, + coordination: { committedCount: 0, fence: "all_target_versions_before_first_write" }, + }); + expect(editCalls).toEqual([]); + expect(cells.get(targetIds[0])!.value).toEqual(before[targetIds[0]].value); + expect(cells.get(targetIds[1])!.value).toBe("+19% human"); + }); + + it("keeps changed, out-of-plan, and empty commit attempts blocked", async () => { + const approvedOperations = [ + OPERATION, + { elementId: "C2", formula: "=B2+1", result: 21, numFmt: "0.00" }, + ]; + const writes: unknown[] = []; + let turn = 0; + const model = scriptedModel(() => { + turn += 1; + if (turn === 1) return { toolCalls: [{ tool: "inspect_workbook", args: { instruction: "repair formulas", artifactId: "sheet-1" } }] }; + if (turn === 2) return { toolCalls: [{ tool: "verify_workbook", args: { instruction: "repair formulas", artifactId: "sheet-1", operations: approvedOperations, afterWrite: false } }] }; + if (turn === 3) return { toolCalls: [{ tool: "write_locked_cells", args: { artifactId: "sheet-1\nwrite_locked_cells", ops: [{ ...OPERATION, formula: "=A2*3" }] } }] }; + if (turn === 4) return { toolCalls: [{ tool: "write_locked_cells", args: { artifactId: "sheet-1", ops: [{ ...OPERATION, elementId: "Z99" }] } }] }; + if (turn === 5) return { toolCalls: [{ tool: "write_locked_cells", args: { artifactId: "sheet-1", ops: [] } }] }; + if (turn === 6) return { toolCalls: [{ tool: "write_locked_cells", args: { artifactId: "sheet-1", ops: [approvedOperations[1]] } }] }; + if (turn === 7) return { toolCalls: [{ tool: "verify_workbook", args: { instruction: "repair formulas", artifactId: "sheet-1", operations: approvedOperations, afterWrite: true } }] }; + return { say: "Only the approved plan was committed.", done: true }; + }); + + const result = await runWorkbookAgent({ + model, + tools: workbookTools({ ok: true }, (args) => writes.push(args)), + maxSteps: 10, + }); + const blockedWrites = result.trace.filter((event) => + event.tool === "write_locked_cells" + && (event.result as { reason?: string }).reason?.includes("write_plan_mismatch")); + + expect(blockedWrites).toHaveLength(3); + expect(writes).toHaveLength(1); + expect(writes[0]).toEqual({ artifactId: "sheet-1", ops: boundOperations(approvedOperations) }); + expect(result.stopReason).toBe("done"); + }); + + it("blocks clean wrong artifacts, malformed batch entries, and excess duplicates", async () => { + const approvedOperations = [ + OPERATION, + { elementId: "C2", formula: "=B2+1", result: 21, numFmt: "0.00" }, + ]; + const invalidArgs = [ + { artifactId: "sheet-2", ops: [approvedOperations[0]] }, + { artifactId: "sheet-1", ops: [approvedOperations[0], null] }, + { artifactId: "sheet-1", ops: [approvedOperations[0], approvedOperations[0]] }, + ]; + + for (const badArgs of invalidArgs) { + const writes: unknown[] = []; + let turn = 0; + const result = await runWorkbookAgent({ + model: scriptedModel(() => { + turn += 1; + if (turn === 1) return { toolCalls: [{ tool: "inspect_workbook", args: { instruction: "repair formulas", artifactId: "sheet-1" } }] }; + if (turn === 2) return { toolCalls: [{ tool: "verify_workbook", args: { instruction: "repair formulas", artifactId: "sheet-1", operations: approvedOperations, afterWrite: false } }] }; + if (turn === 3) return { toolCalls: [{ tool: "write_locked_cells", args: badArgs }] }; + if (turn === 4) return { toolCalls: [{ tool: "write_locked_cells", args: { artifactId: "sheet-1", ops: [approvedOperations[1]] } }] }; + if (turn === 5) return { toolCalls: [{ tool: "verify_workbook", args: { instruction: "repair formulas", artifactId: "sheet-1", operations: approvedOperations, afterWrite: true } }] }; + return { say: "Verified.", done: true }; + }), + tools: workbookTools({ ok: true }, (args) => writes.push(args)), + }); + + expect(result.trace.filter((event) => + (event.result as { reason?: string }).reason?.includes("write_plan_mismatch"))).toHaveLength(1); + expect(writes).toEqual([{ artifactId: "sheet-1", ops: boundOperations(approvedOperations) }]); + expect(result.stopReason).toBe("done"); + } + }); + + it("persists a bound approved plan and reconstructs pending verification after a workflow slice resume", async () => { + const operations = [ + OPERATION, + { elementId: "C2", formula: "=B2+1", result: 21, numFmt: "0.00" }, + ]; + const initialMessages: AgentMessage[] = [ + { role: "user", content: "Complete this SpreadsheetBench workbook formula task." }, + { role: "assistant", content: "", toolCalls: [{ id: "inspect-1", tool: "inspect_workbook", args: { instruction: "fill formulas", artifactId: "sheet-1" } }] }, + { role: "tool", toolCallId: "inspect-1", toolName: "inspect_workbook", content: JSON.stringify({ ok: true, artifactId: "sheet-1" }) }, + { role: "assistant", content: "", toolCalls: [{ id: "preflight-1", tool: "verify_workbook", args: { instruction: "fill formulas", artifactId: "sheet-1", operations, afterWrite: false } }] }, + { role: "tool", toolCallId: "preflight-1", toolName: "verify_workbook", content: JSON.stringify(preflightResult("sheet-1", operations)) }, + ]; + const writes: unknown[] = []; + const firstSlice = await runWorkbookAgent({ + model: scriptedModel(() => ({ toolCalls: [{ tool: "write_locked_cells", args: { ops: [operations[0]] } }] })), + initialMessages, + tools: workbookTools({ ok: true }, (args) => writes.push(args)), + maxSteps: 1, + }); + let resumedTurn = 0; + const result = await runWorkbookAgent({ + model: scriptedModel(() => { + resumedTurn += 1; + if (resumedTurn === 1) return { say: "Done before checking.", done: true }; + if (resumedTurn === 2) return { toolCalls: [{ tool: "verify_workbook", args: { instruction: "fill formulas", artifactId: "sheet-1", operations, afterWrite: true } }] }; + return { say: "Resumed plan committed and verified.", done: true }; + }), + initialMessages: firstSlice.messages, + }); + + expect(writes).toEqual([{ artifactId: "sheet-1", ops: boundOperations(operations) }]); + expect(firstSlice.messages.some((message) => + message.role === "assistant" + && message.toolCalls?.some((call) => call.tool === "write_locked_cells" && (call.args.ops as unknown[])?.length === 2))).toBe(true); + expect(result.trace.map((event) => event.tool)).toEqual(["verify_workbook"]); + expect(result.messages.some((message) => message.role === "user" && message.content.includes("POST_WRITE_VERIFICATION_REQUIRED"))).toBe(true); + expect(result.stopReason).toBe("done"); + }); + + it("preserves exact evidence-bearing result writes unchanged", async () => { + const resultWrite = { + artifactId: "sheet-1", + elementId: "B2", + baseVersion: 7, + formula: "=A2*2", + result: 20, + value: 20, + numFmt: "0.00", + status: "complete", + confidence: 0.95, + evidence: [{ source: "fixture", quote: "A2 is 10" }], + }; + const observed: unknown[] = []; + const resultTool: AgentTool = { + name: "write_locked_cell_result", + description: "Write an evidence-bearing cell result.", + schema: z.object({ + artifactId: z.string().optional(), + elementId: z.string(), + baseVersion: z.number().int(), + formula: z.string().optional(), + result: z.unknown().optional(), + value: z.unknown(), + numFmt: z.string().optional(), + status: z.string(), + confidence: z.number().optional(), + evidence: z.array(z.object({ source: z.string(), quote: z.string() })), + }), + execute: async (args: unknown) => { + observed.push(args); + return { ok: true }; + }, + }; + let turn = 0; + const result = await runWorkbookAgent({ + model: scriptedModel(() => { + turn += 1; + if (turn === 1) return { toolCalls: [{ tool: "inspect_workbook", args: { instruction: "repair formulas", artifactId: "sheet-1" } }] }; + if (turn === 2) return { toolCalls: [{ tool: "verify_workbook", args: { instruction: "repair formulas", artifactId: "sheet-1", operations: [{ ...OPERATION, baseVersion: 7 }], afterWrite: false } }] }; + if (turn === 3) return { toolCalls: [{ tool: "write_locked_cell_result", args: resultWrite }] }; + if (turn === 4) return { toolCalls: [{ tool: "verify_workbook", args: { instruction: "repair formulas", artifactId: "sheet-1", operations: [{ ...OPERATION, baseVersion: 7 }], afterWrite: true } }] }; + return { say: "Evidence-bearing result verified.", done: true }; + }), + tools: [...workbookTools(), resultTool], + }); + + expect(observed).toEqual([resultWrite]); + expect(result.trace.find((event) => event.tool === "write_locked_cell_result")?.args).toEqual(resultWrite); + expect(result.stopReason).toBe("done"); + }); + + it("blocks an artifact-ambiguous subset shared by two approved plans", async () => { + const operations = [OPERATION, { elementId: "C2", formula: "=B2+1", result: 21, numFmt: "0.00" }]; + const initialMessages: AgentMessage[] = []; + for (const artifactId of ["sheet-1", "sheet-2"]) { + initialMessages.push( + { role: "assistant", content: "", toolCalls: [{ id: `inspect-${artifactId}`, tool: "inspect_workbook", args: { instruction: "fill formulas", artifactId } }] }, + { role: "tool", toolCallId: `inspect-${artifactId}`, toolName: "inspect_workbook", content: JSON.stringify({ ok: true, artifactId }) }, + { role: "assistant", content: "", toolCalls: [{ id: `preflight-${artifactId}`, tool: "verify_workbook", args: { instruction: "fill formulas", artifactId, operations, afterWrite: false } }] }, + { role: "tool", toolCallId: `preflight-${artifactId}`, toolName: "verify_workbook", content: JSON.stringify(preflightResult(artifactId, operations)) }, + ); + } + + const result = await runWorkbookAgent({ + model: scriptedModel(() => ({ toolCalls: [{ tool: "write_locked_cells", args: { ops: [operations[0]] } }] })), + initialMessages, + maxSteps: 1, + }); + + expect(result.trace[0]).toMatchObject({ + tool: "write_locked_cells", + result: { ok: false, error: "tool_blocked", metadata: { stage: "write_plan_mismatch", approvedPlanCount: 2, bindingMatchCount: 2 } }, + }); + }); + + it("bounds mismatch metadata for large approved plans", async () => { + const operations = Array.from({ length: 46 }, (_, index) => ({ elementId: `B${index + 2}`, value: index + 2 })); + let turn = 0; + const result = await runWorkbookAgent({ + model: scriptedModel(() => { + turn += 1; + if (turn === 1) return { toolCalls: [{ tool: "inspect_workbook", args: { instruction: "fill values", artifactId: "sheet-1" } }] }; + if (turn === 2) return { toolCalls: [{ tool: "verify_workbook", args: { instruction: "fill values", artifactId: "sheet-1", operations, afterWrite: false } }] }; + if (turn === 3) return { toolCalls: [{ tool: "write_locked_cells", args: { artifactId: "sheet-1", ops: [{ elementId: "Z99", value: 99 }] } }] }; + if (turn === 4) return { toolCalls: [{ tool: "write_locked_cells", args: { artifactId: "sheet-1", ops: [operations[0]] } }] }; + if (turn === 5) return { toolCalls: [{ tool: "verify_workbook", args: { instruction: "fill values", artifactId: "sheet-1", operations, afterWrite: true } }] }; + return { say: "Verified.", done: true }; + }), + maxSteps: 8, + }); + const mismatch = result.trace.find((event) => + (event.result as { reason?: string }).reason?.includes("write_plan_mismatch")); + const metadata = (mismatch?.result as { metadata?: Record }).metadata; + + expect(metadata?.approvedTargets).toHaveLength(8); + expect(metadata?.approvedTargetsOmitted).toBe(38); + expect(metadata).not.toHaveProperty("approvedWrite"); + expect(result.stopReason).toBe("done"); + }); + it("reconstructs a pending post-write verification from durable resume messages", async () => { const initialMessages: AgentMessage[] = [ { role: "user", content: "Complete this SpreadsheetBench workbook formula task." }, { role: "assistant", content: "", toolCalls: [{ id: "inspect-1", tool: "inspect_workbook", args: { instruction: "fill formulas", artifactId: "sheet-1" } }] }, { role: "tool", toolCallId: "inspect-1", toolName: "inspect_workbook", content: JSON.stringify({ ok: true, artifactId: "sheet-1" }) }, { role: "assistant", content: "", toolCalls: [{ id: "preflight-1", tool: "verify_workbook", args: { instruction: "fill formulas", artifactId: "sheet-1", operations: [OPERATION], afterWrite: false } }] }, - { role: "tool", toolCallId: "preflight-1", toolName: "verify_workbook", content: JSON.stringify({ ok: true, status: "passed", artifactId: "sheet-1", phase: "preflight" }) }, - { role: "assistant", content: "", toolCalls: [{ id: "write-1", tool: "write_locked_cells", args: { artifactId: "sheet-1", ops: [OPERATION] } }] }, + { role: "tool", toolCallId: "preflight-1", toolName: "verify_workbook", content: JSON.stringify(preflightResult("sheet-1", [OPERATION])) }, + { role: "assistant", content: "", toolCalls: [{ id: "write-1", tool: "write_locked_cells", args: { artifactId: "sheet-1", ops: boundOperations([OPERATION]) } }] }, { role: "tool", toolCallId: "write-1", toolName: "write_locked_cells", content: JSON.stringify({ ok: true }) }, ]; let turn = 0; From b087187ce76aedc92fa91fefb56811094e319196 Mon Sep 17 00:00:00 2001 From: homen Date: Tue, 14 Jul 2026 19:00:29 -0700 Subject: [PATCH 02/12] fix(nodeagent): harden durable workbook execution convex: fence durable workbook jobs and persist exact trace accounting src: enforce routed spend budgets and correlate live telemetry tests: cover lease, retry, trace, routing, and UI regressions --- convex/agentJobRunner.ts | 205 +++++++++++++---- convex/agentJobs.ts | 66 +++++- convex/agentRuns.ts | 294 ++++++++++++++++++++++++ convex/agentStepChain.ts | 171 ++++++++++++++ convex/agentStepJournal.ts | 38 ++- convex/agentStepJournalClient.ts | 32 ++- convex/agentSteps.ts | 74 +++--- convex/agentWorkflows.ts | 15 +- convex/schema.ts | 9 +- src/app/store.tsx | 70 +++++- src/nodeagent/core/runtime.ts | 22 +- src/nodeagent/core/types.ts | 2 + src/nodeagent/models/convexModel.ts | 157 +++++++++---- src/nodeagent/models/qualityFailover.ts | 10 +- src/ui/RoomShell.tsx | 23 +- tests/agentJobsRuntime.test.ts | 289 ++++++++++++++++++++++- tests/agentJobsSource.test.ts | 33 ++- tests/convexModelFreeAutoMode.test.ts | 98 +++++++- tests/gatewayAndJournal.test.ts | 81 ++++++- tests/idempotencyRuntime.test.ts | 241 +++++++++++++++++++ tests/qualityFailover.test.ts | 21 ++ tests/roomShellTelemetry.test.ts | 113 ++++++++- 22 files changed, 1877 insertions(+), 187 deletions(-) create mode 100644 convex/agentStepChain.ts diff --git a/convex/agentJobRunner.ts b/convex/agentJobRunner.ts index 5c02a5d3..ecbbe728 100644 --- a/convex/agentJobRunner.ts +++ b/convex/agentJobRunner.ts @@ -29,7 +29,7 @@ import type { AgentMessage, AgentModelRouteState, AgentResult, AgentTraceEvent, import type { AgentStreamEventDraft } from "../src/nodeagent/core/stream"; import type { EvidenceState, FrameDelta, ReasoningFrame, ReasoningFrameStatus } from "../src/nodeagent/core/reasoningFrames"; import type { Actor } from "../src/engine/types"; -import { journalSliceKey } from "../src/nodeagent/core/journal"; +import { journalSliceKey, stableJournalHash } from "../src/nodeagent/core/journal"; import { FREE_FILE_EGRESS_BLOCK_REASON, freeFileEgressPromotionAllowed, @@ -59,8 +59,7 @@ const agentJobsFinishSliceRef = makeFunctionReference<"mutation">("agentJobs:fin const agentJobsCompleteDeterministicBenchmarkSliceRef = makeFunctionReference<"mutation">("agentJobs:completeDeterministicBenchmarkSlice") as any; const agentJobsRecordLiveOperationRef = makeFunctionReference<"mutation">("agentJobs:recordLiveOperation") as any; const agentJobsRecordStreamEventRef = makeFunctionReference<"mutation">("agentJobs:recordStreamEvent") as any; -const agentRunsRecordRef = makeFunctionReference<"mutation">("agentRuns:record") as any; -const agentStepsRecordRef = makeFunctionReference<"mutation">("agentSteps:record") as any; +const agentRunsRecordJournaledRef = makeFunctionReference<"mutation">("agentRuns:recordJournaled") as any; const artifactsListForRoomRef = makeFunctionReference<"query">("artifacts:listForRoom") as any; const streamingEnsurePublicAgentJobStreamRef = makeFunctionReference<"mutation">("streaming:ensurePublicAgentJobStream") as any; const streamingAppendPublicAgentJobStreamChunkRef = makeFunctionReference<"mutation">("streaming:appendPublicAgentJobStreamChunk") as any; @@ -120,6 +119,12 @@ type ClaimedReasoningFrame = { type RunTelemetry = { ms: number; + modelCalls: number; + toolCalls: number; + inputTokens: number; + outputTokens: number; + cachedInputTokens: number; + cacheCreationInputTokens: number; costUsd: number; costKind: "exact" | "estimated"; }; @@ -127,8 +132,16 @@ type RunTelemetry = { type RunRecord = { runId: Id<"agentRuns">; telemetry: RunTelemetry; + jobAccounting: Omit; }; +class RunPersistenceError extends Error { + constructor(readonly cause: unknown) { + super(`run_persistence_failed: ${errorText(cause)}`); + this.name = "RunPersistenceError"; + } +} + type PublicAgentJobStream = { streamId: string; clientMsgId: string; @@ -599,22 +612,24 @@ export const runFreeAutoJobSlice = internalAction({ const settleStreamEventWrites = async () => { await Promise.allSettled(streamEventWrites); }; + const modelJournalSliceKey = journalSliceKey({ + entrypoint, + jobId: String(claimed.jobId), + artifactId: String(claimed.artifactId), + frameId: activeFrame?.frameId, + goal: claimed.goal, + mode: claimed.mode ?? "variance", + modelPolicy: resolvedModelPolicy, + runtimeProfile: claimed.runtimeProfile, + cursor: claimed.cursor ?? null, + handoff: claimed.handoff ?? null, + maxSteps, + }); const modelJournal = makeConvexStepJournal({ ctx, jobId: claimed.jobId, - sliceKey: journalSliceKey({ - entrypoint, - jobId: String(claimed.jobId), - artifactId: String(claimed.artifactId), - frameId: activeFrame?.frameId, - goal: claimed.goal, - mode: claimed.mode ?? "variance", - modelPolicy: resolvedModelPolicy, - runtimeProfile: claimed.runtimeProfile, - cursor: claimed.cursor ?? null, - handoff: claimed.handoff ?? null, - maxSteps, - }), + leaseId, + sliceKey: modelJournalSliceKey, modelName: () => phaseAwareModel.name, }); let publicStream: PublicAgentJobStream | undefined; @@ -710,11 +725,30 @@ export const runFreeAutoJobSlice = internalAction({ deadlineAt, handoff: result.handoff, }; - const runId = await ctx.runMutation(agentRunsRecordRef, telemetry); - const steps = result.trace.map(traceStep); + const journalClaims = modelJournal.accountingClaims(); + const traceSteps = result.trace.map(traceStep); + if (traceSteps.length === 0 && !extraStep) { + const syntheticResult = result.finalText?.trim() + || (result.handoff ? JSON.stringify(result.handoff) : result.stopReason); + traceSteps.push({ + idx: 0, + tool: "model_result", + args: cap(JSON.stringify({ + stopReason: result.stopReason, + steps: result.steps, + modelCalls: result.usage.modelCalls ?? 0, + })), + result: cap(syntheticResult), + status: result.stopReason === "error" ? "error" as const : "ok" as const, + ms, + elementId: undefined, + affectedObjectIds: undefined, + mutationReceiptIds: undefined, + }); + } if (extraStep) { - steps.push({ - idx: steps.length, + traceSteps.push({ + idx: traceSteps.length, tool: extraStep.tool, args: cap(JSON.stringify({ jobId: String(claimed.jobId), attempt: claimed.attempt })), result: cap(extraStep.result), @@ -725,14 +759,58 @@ export const runFreeAutoJobSlice = internalAction({ mutationReceiptIds: undefined, }); } - await ctx.runMutation(agentStepsRecordRef, { - jobId: claimed.jobId, + const recordKey = `durable-run:${stableJournalHash({ + jobId: String(claimed.jobId), + leaseId, + attempt: claimed.attempt, + kind: extraStep ? "error" : "result", + telemetry, + journalClaims, + traceSteps, + })}`; + const payload = { + ...telemetry, + leaseId, + attempt: claimed.attempt, + recordKey, + journalSliceKey: modelJournalSliceKey, + journalClaims, + traceSteps, + }; + let recorded: { + runId: Id<"agentRuns">; + accounting: Omit; + jobAccounting: Omit; + }; + try { + recorded = await ctx.runMutation(agentRunsRecordJournaledRef, payload) as typeof recorded; + } catch { + // A mutation can commit before its response is lost. One exact, idempotent replay recovers + // that result; a second failure is an integrity blocker and must not fall through to a + // different zero-usage run. + try { + recorded = await ctx.runMutation(agentRunsRecordJournaledRef, payload) as typeof recorded; + } catch (retryError) { + throw new RunPersistenceError(retryError); + } + } + const runId = recorded.runId; + return { runId, - roomId: claimed.roomId, - agentId: actor.id, - steps, - }); - return { runId, telemetry: { ...telemetry, costKind } }; + telemetry: { ms, ...recorded.accounting }, + jobAccounting: recorded.jobAccounting, + }; + }; + + let persistedRun: RunRecord | undefined; + const finishSlice = async (payload: Record) => { + try { + return await ctx.runMutation(agentJobsFinishSliceRef, payload) as { ok: boolean; reason?: string; reused?: boolean }; + } catch { + // Recover a committed mutation whose response was lost. finishSlice recognizes the exact + // persisted attempt and returns reused without applying accounting twice. + return await ctx.runMutation(agentJobsFinishSliceRef, payload) as { ok: boolean; reason?: string; reused?: boolean }; + } }; try { @@ -895,7 +973,8 @@ export const runFreeAutoJobSlice = internalAction({ hooks: workbookWorkflowHooks, }); } - const { runId, telemetry } = await recordRun(result); + persistedRun = await recordRun(result); + const { runId, telemetry, jobAccounting } = persistedRun; const done = result.stopReason === "done" && !result.exhausted; if (handledByHmdaBenchmark && done) { const terminalText = result.finalText || publicStreamText || ""; @@ -982,7 +1061,7 @@ export const runFreeAutoJobSlice = internalAction({ kind: "model_call", name: phaseAwareModel.name, status: "completed", - countDelta: result.usage.modelCalls, + countDelta: telemetry.modelCalls, completedAt: Date.now(), }); await recordLiveOperation({ @@ -995,7 +1074,7 @@ export const runFreeAutoJobSlice = internalAction({ await Promise.allSettled(liveWrites); await settleStreamEventWrites(); - await ctx.runMutation(agentJobsFinishSliceRef, clean({ + const finishResult = await finishSlice(clean({ jobId: claimed.jobId, leaseId, attempt: claimed.attempt, @@ -1003,14 +1082,15 @@ export const runFreeAutoJobSlice = internalAction({ resolvedModel: phaseAwareModel.name, stopReason: result.stopReason, ms: telemetry.ms, - inputTokens: result.usage.inputTokens, - outputTokens: result.usage.outputTokens, - cachedInputTokens: result.usage.cachedInputTokens ?? 0, - cacheCreationInputTokens: result.usage.cacheCreationInputTokens ?? 0, + inputTokens: telemetry.inputTokens, + outputTokens: telemetry.outputTokens, + cachedInputTokens: telemetry.cachedInputTokens, + cacheCreationInputTokens: telemetry.cacheCreationInputTokens, costUsd: telemetry.costUsd, costKind: telemetry.costKind, - modelCalls: result.usage.modelCalls, - toolCalls: modelToolCallCount(result.trace), + modelCalls: telemetry.modelCalls, + toolCalls: telemetry.toolCalls, + accountingTotals: jobAccounting, runId, handoff: result.handoff, cursor, @@ -1034,6 +1114,14 @@ export const runFreeAutoJobSlice = internalAction({ runtimeError: frameReceipt.runtimeError, } : undefined, })); + if (!finishResult.ok) { + return { + ok: false as const, + retrying: true, + error: `agent_job_finish_rejected:${finishResult.reason ?? "unknown"}`, + runId, + }; + } return { ok: true as const, done, stopReason: result.stopReason, runId }; } catch (error) { @@ -1059,10 +1147,26 @@ export const runFreeAutoJobSlice = internalAction({ messages: [], usage: { inputTokens: 0, outputTokens: 0, modelCalls: 0 }, }; - const { runId, telemetry } = await recordRun(fallback, { tool: "job_error", result: errorText(rootError) }); + if (rootError instanceof RunPersistenceError) { + await settlePublicStreamWrites(); + void recordStreamEvent({ + kind: "error", + status: "failed", + title: "Agent run persistence failed", + error: errorText(rootError), + metadata: { attempt: claimed.attempt, integrityBlocked: true }, + createdAt: Date.now(), + }); + await settleStreamEventWrites(); + return { ok: false as const, retrying: true, error: errorText(rootError) }; + } + const runAlreadyPersisted = persistedRun !== undefined; + const recordedFailure = persistedRun ?? await recordRun(fallback, { tool: "job_error", result: errorText(rootError) }); + persistedRun = recordedFailure; + const { runId, telemetry, jobAccounting } = recordedFailure; const nonRetryableReason = providerNonRetryableReason(rootError); const retryable = !isProviderNonRetryableError(rootError); - const canRetry = retryable && claimed.attempt < claimed.maxAttempts; + const canRetry = !runAlreadyPersisted && retryable && claimed.attempt < claimed.maxAttempts; const delayMs = canRetry ? backoffMs(claimed.attempt) : undefined; const scheduledNextAt = delayMs ? Date.now() + delayMs : undefined; const activeFrameId = claimed.activeReasoningFrame?.frameId; @@ -1097,7 +1201,7 @@ export const runFreeAutoJobSlice = internalAction({ kind: "model_call", name: phaseAwareModel.name, status: "failed", - countDelta: fallback.usage.modelCalls, + countDelta: runAlreadyPersisted ? 0 : telemetry.modelCalls, completedAt: Date.now(), }); await recordLiveOperation({ @@ -1110,7 +1214,7 @@ export const runFreeAutoJobSlice = internalAction({ await Promise.allSettled(liveWrites); await settleStreamEventWrites(); - await ctx.runMutation(agentJobsFinishSliceRef, clean({ + const finishResult = await finishSlice(clean({ jobId: claimed.jobId, leaseId, attempt: claimed.attempt, @@ -1118,14 +1222,15 @@ export const runFreeAutoJobSlice = internalAction({ resolvedModel: phaseAwareModel.name, stopReason: fallback.stopReason, ms: telemetry.ms, - inputTokens: fallback.usage.inputTokens, - outputTokens: fallback.usage.outputTokens, - cachedInputTokens: fallback.usage.cachedInputTokens ?? 0, - cacheCreationInputTokens: fallback.usage.cacheCreationInputTokens ?? 0, + inputTokens: telemetry.inputTokens, + outputTokens: telemetry.outputTokens, + cachedInputTokens: telemetry.cachedInputTokens, + cacheCreationInputTokens: telemetry.cacheCreationInputTokens, costUsd: telemetry.costUsd, costKind: telemetry.costKind, - modelCalls: fallback.usage.modelCalls, - toolCalls: modelToolCallCount(fallback.trace), + modelCalls: telemetry.modelCalls, + toolCalls: telemetry.toolCalls, + accountingTotals: jobAccounting, runId, error: errorText(rootError), handoff: fallback.handoff, @@ -1134,6 +1239,14 @@ export const runFreeAutoJobSlice = internalAction({ frameId: activeFrameId, frameStatus: canRetry ? "pending" : "failed", })); + if (!finishResult.ok) { + return { + ok: false as const, + retrying: true, + error: `${errorText(rootError)}; agent_job_finish_rejected:${finishResult.reason ?? "unknown"}`, + runId, + }; + } return { ok: false as const, retrying: canRetry, error: errorText(rootError), runId }; } diff --git a/convex/agentJobs.ts b/convex/agentJobs.ts index 052da2f1..62d3d51f 100644 --- a/convex/agentJobs.ts +++ b/convex/agentJobs.ts @@ -28,6 +28,16 @@ function companyKeyOf(name: string): string { const attemptStatusV = v.union(v.literal("completed"), v.literal("handoff"), v.literal("retrying"), v.literal("blocked"), v.literal("failed")); const costKindV = v.union(v.literal("exact"), v.literal("estimated")); +const jobAccountingTotalsV = v.object({ + modelCalls: v.number(), + toolCalls: v.number(), + inputTokens: v.number(), + outputTokens: v.number(), + cachedInputTokens: v.number(), + cacheCreationInputTokens: v.number(), + costUsd: v.number(), + costKind: costKindV, +}); const terminalStatuses = new Set(["completed", "failed", "blocked", "cancelled"]); const entrypointV = v.union( v.literal("public_ask"), @@ -1162,7 +1172,7 @@ export const finishInteractive = internalMutation({ handler: async (ctx, a) => { const job = await ctx.db.get(a.jobId); if (!job) return { ok: false as const, reason: "job_not_found" as const }; - if (terminalStatuses.has(job.status) && job.latestRunId) return { ok: true as const, terminal: true as const }; + if (terminalStatuses.has(job.status)) return { ok: true as const, terminal: true as const }; const modelCalls = persistedCallCount(a.modelCalls, 0, "modelCalls"); const toolCalls = persistedCallCount(a.toolCalls, 0, "toolCalls"); const costKind: CostKind = a.costKind ?? "estimated"; @@ -2961,7 +2971,12 @@ export const recordWorkflowComplete = internalMutation({ if (terminalStatuses.has(job.status)) { return { ok: true as const, terminal: true as const }; } - if (resultKind !== "success" && job.status === "running" && job.attempts > 1) { + const concurrentSliceOwnsLease = resultKind === "failed" + && error?.includes("agent_slice_failed:not_claimed") + && job.status === "running" + && Boolean(job.leaseId) + && Boolean(job.leaseUntil && job.leaseUntil > Date.now()); + if (concurrentSliceOwnsLease) { return { ok: true as const, terminal: false as const, superseded: true as const }; } // P0: Passive research jobs must not requeue on spend_budget/rate_limit failures. @@ -3317,6 +3332,7 @@ export const finishSlice = internalMutation({ costKind: v.optional(costKindV), modelCalls: v.optional(v.number()), toolCalls: v.optional(v.number()), + accountingTotals: v.optional(jobAccountingTotalsV), runId: v.optional(v.id("agentRuns")), error: v.optional(v.string()), handoff: v.optional(v.any()), @@ -3332,14 +3348,36 @@ export const finishSlice = internalMutation({ handler: async (ctx, a) => { const job = await ctx.db.get(a.jobId); if (!job) return { ok: false as const, reason: "job_not_found" as const }; - if (job.leaseId !== a.leaseId) return { ok: false as const, reason: "lease_mismatch" as const }; // Optional fallbacks preserve callers deployed before exact accounting. New callers pass // explicit values, including zero, which are never replaced by the legacy estimates. const modelCalls = persistedCallCount(a.modelCalls, 1, "modelCalls"); const toolCalls = persistedCallCount(a.toolCalls, a.inputTokens || a.outputTokens ? 1 : 0, "toolCalls"); const costKind: CostKind = a.costKind ?? "estimated"; - const accounting = await loadJobAccountingBaseline(ctx, a.jobId, job); + if (a.runId) { + const priorAttempt = await ctx.db.query("agentJobAttempts") + .withIndex("by_job", (query) => query.eq("jobId", a.jobId).eq("attempt", a.attempt)) + .order("desc") + .first(); + const exactReplay = priorAttempt + && String(priorAttempt.runId ?? "") === String(a.runId) + && priorAttempt.status === a.status + && priorAttempt.resolvedModel === a.resolvedModel + && priorAttempt.stopReason === a.stopReason + && priorAttempt.inputTokens === a.inputTokens + && priorAttempt.outputTokens === a.outputTokens + && (priorAttempt.cachedInputTokens ?? 0) === (a.cachedInputTokens ?? 0) + && (priorAttempt.cacheCreationInputTokens ?? 0) === (a.cacheCreationInputTokens ?? 0) + && priorAttempt.costUsd === a.costUsd + && (priorAttempt.costKind ?? "estimated") === costKind + && (priorAttempt.modelCalls ?? 1) === modelCalls + && (priorAttempt.toolCalls ?? 0) === toolCalls; + if (exactReplay) return { ok: true as const, reused: true as const }; + } const now = Date.now(); + if (job.leaseId !== a.leaseId) return { ok: false as const, reason: "lease_mismatch" as const }; + if (terminalStatuses.has(job.status)) return { ok: false as const, reason: "job_terminal" as const }; + if (!job.leaseUntil || job.leaseUntil <= now) return { ok: false as const, reason: "lease_expired" as const }; + const accounting = await loadJobAccountingBaseline(ctx, a.jobId, job); await ctx.db.insert("agentJobAttempts", clean({ jobId: a.jobId, runId: a.runId, @@ -3370,10 +3408,7 @@ export const finishSlice = internalMutation({ a.status === "retrying" ? "retrying" : "paused"; - const patch: Record = { - status: nextStatus, - leaseId: "", - leaseUntil: 0, + const incremental = { modelCallCount: accounting.modelCallCount + modelCalls, toolCallCount: accounting.toolCallCount + toolCalls, inputTokens: accounting.inputTokens + a.inputTokens, @@ -3381,7 +3416,20 @@ export const finishSlice = internalMutation({ cachedInputTokens: accounting.cachedInputTokens + (a.cachedInputTokens ?? 0), cacheCreationInputTokens: accounting.cacheCreationInputTokens + (a.cacheCreationInputTokens ?? 0), costUsd: accounting.costUsd + a.costUsd, - costKind: aggregateCostKind(accounting.costKind, costKind), + }; + const totals = a.accountingTotals; + const patch: Record = { + status: nextStatus, + leaseId: "", + leaseUntil: 0, + modelCallCount: totals ? Math.max(accounting.modelCallCount, totals.modelCalls) : incremental.modelCallCount, + toolCallCount: totals ? Math.max(accounting.toolCallCount, totals.toolCalls) : incremental.toolCallCount, + inputTokens: totals ? Math.max(accounting.inputTokens, totals.inputTokens) : incremental.inputTokens, + outputTokens: totals ? Math.max(accounting.outputTokens, totals.outputTokens) : incremental.outputTokens, + cachedInputTokens: totals ? Math.max(accounting.cachedInputTokens, totals.cachedInputTokens) : incremental.cachedInputTokens, + cacheCreationInputTokens: totals ? Math.max(accounting.cacheCreationInputTokens, totals.cacheCreationInputTokens) : incremental.cacheCreationInputTokens, + costUsd: totals ? Math.max(accounting.costUsd, totals.costUsd) : incremental.costUsd, + costKind: aggregateCostKind(aggregateCostKind(accounting.costKind, costKind), totals?.costKind ?? "exact"), mutationCount: (job.mutationCount ?? 0) + 1, updatedAt: now, }; diff --git a/convex/agentRuns.ts b/convex/agentRuns.ts index 29b9021d..c82e5840 100644 --- a/convex/agentRuns.ts +++ b/convex/agentRuns.ts @@ -3,8 +3,153 @@ import { v } from "convex/values"; import { internalMutation, internalQuery, query } from "./_generated/server"; import { actorProofV, requireActorProof } from "./lib"; import { findReusableRun } from "../src/nodeagent/core/idempotency"; +import { convexPriceRun } from "../src/nodeagent/models/convexModel"; +import { agentTraceStepV, recordAgentStepChain } from "./agentStepChain"; const costKindV = v.union(v.literal("exact"), v.literal("estimated")); +const journalClaimV = v.object({ + step: v.number(), + outputHash: v.string(), + state: v.union(v.literal("confirmed"), v.literal("pending")), +}); + +type CostKind = "exact" | "estimated"; +type ModelAccounting = { + modelCalls: number; + inputTokens: number; + outputTokens: number; + cachedInputTokens: number; + cacheCreationInputTokens: number; + costUsd: number; + costKind: CostKind; +}; +type JobAccounting = ModelAccounting & { toolCalls: number }; + +const zeroModelAccounting = (): ModelAccounting => ({ + modelCalls: 0, + inputTokens: 0, + outputTokens: 0, + cachedInputTokens: 0, + cacheCreationInputTokens: 0, + costUsd: 0, + costKind: "exact", +}); + +function nonNegative(value: unknown, label: string, fallback = 0): number { + const resolved = value === undefined ? fallback : value; + if (typeof resolved !== "number" || !Number.isFinite(resolved) || resolved < 0) { + throw new Error(`agent_run_invalid_${label}`); + } + return resolved; +} + +function addModelAccounting(target: ModelAccounting, source: ModelAccounting): ModelAccounting { + target.modelCalls += source.modelCalls; + target.inputTokens += source.inputTokens; + target.outputTokens += source.outputTokens; + target.cachedInputTokens += source.cachedInputTokens; + target.cacheCreationInputTokens += source.cacheCreationInputTokens; + target.costUsd += source.costUsd; + if (source.costKind === "estimated") target.costKind = "estimated"; + return target; +} + +function journalRowAccounting(row: any): ModelAccounting { + const usage = row?.result?.usage as Record | undefined; + const inputTokens = nonNegative(usage?.inputTokens, "journal_input_tokens"); + const outputTokens = nonNegative(usage?.outputTokens, "journal_output_tokens"); + const cachedInputTokens = nonNegative(usage?.cachedInputTokens, "journal_cached_input_tokens"); + const cacheCreationInputTokens = nonNegative(usage?.cacheCreationInputTokens, "journal_cache_creation_input_tokens"); + const explicitCost = usage?.costUsd; + const costUsd = explicitCost === undefined + ? convexPriceRun(String(row.model), inputTokens, outputTokens) + : nonNegative(explicitCost, "journal_cost"); + return { + modelCalls: nonNegative(usage?.modelCalls, "journal_model_calls", 1), + inputTokens, + outputTokens, + cachedInputTokens, + cacheCreationInputTokens, + costUsd, + costKind: explicitCost !== undefined && usage?.costKind === "exact" ? "exact" : "estimated", + }; +} + +function subtractJournalUsage(total: ModelAccounting, journal: ModelAccounting): ModelAccounting { + const subtract = (field: keyof Omit, epsilon = 0) => { + const remainder = total[field] - journal[field]; + if (remainder < -epsilon) throw new Error(`agent_run_journal_usage_mismatch:${field}`); + return Math.max(0, remainder); + }; + return { + modelCalls: subtract("modelCalls"), + inputTokens: subtract("inputTokens"), + outputTokens: subtract("outputTokens"), + cachedInputTokens: subtract("cachedInputTokens"), + cacheCreationInputTokens: subtract("cacheCreationInputTokens"), + costUsd: subtract("costUsd", 1e-9), + costKind: total.costKind, + }; +} + +function hasModelAccounting(value: ModelAccounting): boolean { + return value.modelCalls > 0 + || value.inputTokens > 0 + || value.outputTokens > 0 + || value.cachedInputTokens > 0 + || value.cacheCreationInputTokens > 0 + || value.costUsd > 0; +} + +function runRowAccounting(row: any): JobAccounting { + const inputTokens = nonNegative(row.inputTokens, "row_input_tokens"); + const outputTokens = nonNegative(row.outputTokens, "row_output_tokens"); + const costUsd = nonNegative(row.costUsd, "row_cost"); + return { + modelCalls: nonNegative(row.modelCalls, "row_model_calls", inputTokens || outputTokens || costUsd ? 1 : 0), + toolCalls: nonNegative(row.toolCalls, "row_tool_calls"), + inputTokens, + outputTokens, + cachedInputTokens: nonNegative(row.cachedInputTokens, "row_cached_input_tokens"), + cacheCreationInputTokens: nonNegative(row.cacheCreationInputTokens, "row_cache_creation_input_tokens"), + costUsd, + costKind: row.costKind === "exact" ? "exact" : "estimated", + }; +} + +const MAX_JOB_ACCOUNTING_RUNS = 100; + +async function loadJobRunAccounting(ctx: any, jobId: any): Promise { + const rows = await ctx.db.query("agentRuns") + .withIndex("by_job", (q: any) => q.eq("jobId", jobId)) + .order("asc") + .take(MAX_JOB_ACCOUNTING_RUNS + 1); + if (rows.length > MAX_JOB_ACCOUNTING_RUNS) throw new Error("agent_job_run_accounting_overflow"); + const total: JobAccounting = { ...zeroModelAccounting(), toolCalls: 0 }; + for (const row of rows) { + const accounting = runRowAccounting(row); + addModelAccounting(total, accounting); + total.toolCalls += accounting.toolCalls; + } + return total; +} + +async function syncJobRunAccounting(ctx: any, jobId: any): Promise { + const total = await loadJobRunAccounting(ctx, jobId); + const job = await ctx.db.get(jobId); + if (!job) throw new Error("job_not_found"); + await ctx.db.patch(jobId, { + modelCallCount: Math.max(job.modelCallCount ?? 0, total.modelCalls), + toolCallCount: Math.max(job.toolCallCount ?? 0, total.toolCalls), + inputTokens: Math.max(job.inputTokens ?? 0, total.inputTokens), + outputTokens: Math.max(job.outputTokens ?? 0, total.outputTokens), + cachedInputTokens: Math.max(job.cachedInputTokens ?? 0, total.cachedInputTokens), + cacheCreationInputTokens: Math.max(job.cacheCreationInputTokens ?? 0, total.cacheCreationInputTokens), + costUsd: Math.max(job.costUsd ?? 0, total.costUsd), + costKind: job.costKind === "estimated" || total.costKind === "estimated" ? "estimated" : "exact", + }); + return total; +} /** Claim a run row up-front (idempotency layer 1): a concurrent duplicate sees this in-flight row * (no stopReason yet) via byKey and bails before racing the same locks/CAS. Finished by `finish`. */ @@ -63,6 +208,7 @@ export const byKey = internalQuery({ export const record = internalMutation({ args: { jobId: v.optional(v.id("agentJobs")), + idempotencyKey: v.optional(v.string()), roomId: v.id("rooms"), agentId: v.string(), model: v.string(), goal: v.string(), steps: v.number(), modelCalls: v.optional(v.number()), toolCalls: v.number(), conflictsSurvived: v.number(), inputTokens: v.number(), outputTokens: v.number(), cachedInputTokens: v.optional(v.number()), cacheCreationInputTokens: v.optional(v.number()), @@ -78,6 +224,154 @@ export const record = internalMutation({ }, }); +/** + * Persist a durable-job run while claiming each journaled model response exactly once. + * Replayed journal rows are removed from this run's accounting, while provider failures + * that never reached the journal remain as a per-invocation residual. The claim, run row, + * and cumulative job totals are one serializable transaction. + */ +export const recordJournaled = internalMutation({ + args: { + jobId: v.id("agentJobs"), + leaseId: v.string(), + attempt: v.number(), + recordKey: v.string(), + journalSliceKey: v.string(), + journalClaims: v.array(journalClaimV), + traceSteps: v.array(agentTraceStepV), + roomId: v.id("rooms"), agentId: v.string(), model: v.string(), goal: v.string(), + steps: v.number(), modelCalls: v.number(), toolCalls: v.number(), conflictsSurvived: v.number(), + inputTokens: v.number(), outputTokens: v.number(), cachedInputTokens: v.number(), cacheCreationInputTokens: v.number(), + costUsd: v.number(), costKind: costKindV, ms: v.number(), exhausted: v.boolean(), + stopReason: v.optional(v.string()), remainingMs: v.optional(v.number()), deadlineAt: v.optional(v.number()), handoff: v.optional(v.any()), + }, + handler: async (ctx, a) => { + const job = await ctx.db.get(a.jobId); + if (!job) throw new Error("agent_run_job_not_found"); + if (String(job.roomId) !== String(a.roomId)) throw new Error("agent_run_room_mismatch"); + if (job.attempts !== a.attempt) throw new Error("agent_run_attempt_mismatch"); + if ( + job.status !== "running" + || job.leaseId !== a.leaseId + || !job.leaseUntil + || job.leaseUntil <= Date.now() + ) { + throw new Error("agent_run_lease_invalid"); + } + if (a.traceSteps.length === 0) throw new Error("agent_run_trace_empty"); + + const existing = await ctx.db.query("agentRuns") + .withIndex("by_idempotency", (q) => q.eq("idempotencyKey", a.recordKey)) + .order("desc") + .first(); + if (existing) { + if (String(existing.jobId ?? "") !== String(a.jobId) || String(existing.roomId) !== String(a.roomId)) { + throw new Error("agent_run_record_key_collision"); + } + await recordAgentStepChain(ctx, { + jobId: a.jobId, + runId: existing._id, + roomId: a.roomId, + agentId: a.agentId, + steps: a.traceSteps, + }); + return { + runId: existing._id, + reused: true as const, + accounting: runRowAccounting(existing), + jobAccounting: await syncJobRunAccounting(ctx, a.jobId), + }; + } + + const seenSteps = new Set(); + const journalRows: any[] = []; + for (const claim of a.journalClaims) { + if (seenSteps.has(claim.step)) throw new Error("agent_run_duplicate_journal_claim"); + seenSteps.add(claim.step); + const row = await ctx.db.query("agentModelStepJournal") + .withIndex("by_job_slice_step", (q) => q.eq("jobId", a.jobId).eq("sliceKey", a.journalSliceKey).eq("step", claim.step)) + .order("asc") + .first(); + if (!row || row.outputHash !== claim.outputHash) { + if (claim.state === "confirmed") throw new Error("agent_run_journal_claim_mismatch"); + // A pending claim means journal.record may have failed before commit. Its provider usage + // remains in the aggregate and is accounted below as an unjournaled residual. + continue; + } + journalRows.push(row); + } + + const total: ModelAccounting = { + modelCalls: nonNegative(a.modelCalls, "total_model_calls"), + inputTokens: nonNegative(a.inputTokens, "total_input_tokens"), + outputTokens: nonNegative(a.outputTokens, "total_output_tokens"), + cachedInputTokens: nonNegative(a.cachedInputTokens, "total_cached_input_tokens"), + cacheCreationInputTokens: nonNegative(a.cacheCreationInputTokens, "total_cache_creation_input_tokens"), + costUsd: nonNegative(a.costUsd, "total_cost"), + costKind: a.costKind, + }; + const allJournalAccounting = zeroModelAccounting(); + const newlyClaimedAccounting = zeroModelAccounting(); + const rowsToClaim: any[] = []; + for (const row of journalRows) { + const accounting = journalRowAccounting(row); + addModelAccounting(allJournalAccounting, accounting); + if (!row.accountedRunId) { + addModelAccounting(newlyClaimedAccounting, accounting); + rowsToClaim.push(row); + } + } + const residual = subtractJournalUsage(total, allJournalAccounting); + const accounting = addModelAccounting(zeroModelAccounting(), newlyClaimedAccounting); + if (hasModelAccounting(residual)) addModelAccounting(accounting, residual); + + const doc = { + jobId: a.jobId, + roomId: a.roomId, + agentId: a.agentId, + model: a.model, + goal: a.goal, + steps: a.steps, + traceRecordCount: a.traceSteps.length, + modelCalls: accounting.modelCalls, + toolCalls: a.toolCalls, + conflictsSurvived: a.conflictsSurvived, + inputTokens: accounting.inputTokens, + outputTokens: accounting.outputTokens, + cachedInputTokens: accounting.cachedInputTokens, + cacheCreationInputTokens: accounting.cacheCreationInputTokens, + costUsd: accounting.costUsd, + costKind: accounting.costKind, + ms: a.ms, + exhausted: a.exhausted, + stopReason: a.stopReason, + remainingMs: a.remainingMs, + deadlineAt: a.deadlineAt, + handoff: a.handoff, + idempotencyKey: a.recordKey, + createdAt: Date.now(), + }; + const runId = await ctx.db.insert("agentRuns", doc as any); + await recordAgentStepChain(ctx, { + jobId: a.jobId, + runId, + roomId: a.roomId, + agentId: a.agentId, + steps: a.traceSteps, + }); + const accountingClaimedAt = Date.now(); + for (const row of rowsToClaim) { + await ctx.db.patch(row._id, { accountedRunId: runId, accountingClaimedAt }); + } + return { + runId, + reused: false as const, + accounting: { ...accounting, toolCalls: a.toolCalls }, + jobAccounting: await syncJobRunAccounting(ctx, a.jobId), + }; + }, +}); + /** Production gate: a room's total agent spend (USD) since `since`. Bounded read — the by_room index * is [roomId, createdAt], so a rolling-day window scans only that room's recent runs. The cross-run * cumulative cap that the per-run/per-slice ceilings cannot provide (one run is bounded; the SUM diff --git a/convex/agentStepChain.ts b/convex/agentStepChain.ts new file mode 100644 index 00000000..03adc734 --- /dev/null +++ b/convex/agentStepChain.ts @@ -0,0 +1,171 @@ +import { v } from "convex/values"; +import type { Id } from "./_generated/dataModel"; +import type { MutationCtx } from "./_generated/server"; + +export const agentTraceStepV = v.object({ + idx: v.number(), + tool: v.string(), + args: v.string(), + result: v.string(), + status: v.union(v.literal("ok"), v.literal("conflict"), v.literal("locked"), v.literal("error")), + ms: v.number(), + elementId: v.optional(v.string()), + affectedObjectIds: v.optional(v.array(v.string())), + mutationReceiptIds: v.optional(v.array(v.id("agentMutationReceipts"))), +}); + +export type AgentTraceStepInput = { + idx: number; + tool: string; + args: string; + result: string; + status: "ok" | "conflict" | "locked" | "error"; + ms: number; + elementId?: string; + affectedObjectIds?: string[]; + mutationReceiptIds?: Id<"agentMutationReceipts">[]; +}; + +type ChainIdentity = { + jobId?: Id<"agentJobs">; + runId: Id<"agentRuns">; + roomId: Id<"rooms">; + agentId: string; +}; + +type StoredAgentTraceStepInput = AgentTraceStepInput & ChainIdentity; + +async function sha256hex(value: string): Promise { + const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(value)); + return Array.from(new Uint8Array(digest)).map((byte) => byte.toString(16).padStart(2, "0")).join(""); +} + +function canonical(value: Record): string { + return JSON.stringify(Object.keys(value).sort().reduce>((result, key) => { + result[key] = value[key]; + return result; + }, {})); +} + +function chainCore(identity: ChainIdentity, step: AgentTraceStepInput, prevStepHash: string): Record { + return { + jobId: identity.jobId ? String(identity.jobId) : "", + runId: String(identity.runId), + roomId: String(identity.roomId), + agentId: identity.agentId, + idx: step.idx, + tool: step.tool, + args: step.args, + result: step.result, + status: step.status, + ms: step.ms, + elementId: step.elementId ?? "", + affectedObjectIds: step.affectedObjectIds ?? [], + mutationReceiptIds: (step.mutationReceiptIds ?? []).map(String), + prevStepHash, + }; +} + +export async function recordAgentStepChain( + ctx: MutationCtx, + args: ChainIdentity & { steps: AgentTraceStepInput[] }, +): Promise<{ reused: boolean; inserted: number }> { + if (args.steps.length === 0) throw new Error("agent_step_chain_empty"); + const run = await ctx.db.get(args.runId); + if (!run) throw new Error("agent_run_not_found"); + if (String(run.jobId ?? "") !== String(args.jobId ?? "") + || String(run.roomId) !== String(args.roomId) + || run.agentId !== args.agentId) { + throw new Error("agent_step_chain_identity_mismatch"); + } + if (run.traceRecordCount !== undefined && run.traceRecordCount !== args.steps.length) { + throw new Error("agent_step_chain_count_mismatch"); + } + + const ts = Date.now(); + let prevStepHash = `genesis:${args.runId}`; + const prepared = [] as Array; + for (const step of args.steps) { + const recordHash = await sha256hex(canonical(chainCore(args, step, prevStepHash))); + prepared.push({ + jobId: args.jobId, + runId: args.runId, + roomId: args.roomId, + agentId: args.agentId, + ...step, + ts, + recordHash, + prevStepHash, + }); + prevStepHash = recordHash; + } + + const existing = await ctx.db.query("agentSteps") + .withIndex("by_run", (query) => query.eq("runId", args.runId)) + .order("asc") + .collect(); + if (existing.length > 0) { + const same = existing.length === prepared.length && existing.every((row, index) => { + const expected = prepared[index]; + return row.recordHash === expected.recordHash && row.prevStepHash === expected.prevStepHash; + }); + if (!same) throw new Error("agent_steps_replay_mismatch"); + if (run.traceRecordCount === undefined) await ctx.db.patch(args.runId, { traceRecordCount: prepared.length }); + return { reused: true, inserted: 0 }; + } + + for (const row of prepared) { + await ctx.db.insert("agentSteps", { + jobId: row.jobId, + runId: row.runId, + roomId: row.roomId, + agentId: row.agentId, + idx: row.idx, + tool: row.tool, + args: row.args, + result: row.result, + status: row.status, + ms: row.ms, + elementId: row.elementId, + affectedObjectIds: row.affectedObjectIds, + mutationReceiptIds: row.mutationReceiptIds, + ts: row.ts, + recordHash: row.recordHash, + prevStepHash: row.prevStepHash, + }); + } + if (run.traceRecordCount === undefined) await ctx.db.patch(args.runId, { traceRecordCount: prepared.length }); + return { reused: false, inserted: prepared.length }; +} + +export async function verifyAgentStepChain(args: { + jobId?: Id<"agentJobs">; + runId: Id<"agentRuns">; + roomId: Id<"rooms">; + agentId: string; + expectedCount?: number; + rows: StoredAgentTraceStepInput[]; + hashes: Array<{ recordHash: string; prevStepHash: string }>; +}): Promise<{ valid: true; steps: number } | { valid: false; brokenAt?: number; steps?: number; reason: string }> { + if (args.expectedCount === undefined) return { valid: false, steps: args.rows.length, reason: "trace_count_unavailable" }; + if (args.expectedCount === 0) return { valid: false, steps: args.rows.length, reason: "trace_empty" }; + if (args.rows.length !== args.expectedCount || args.hashes.length !== args.expectedCount) { + return { valid: false, steps: args.rows.length, reason: "trace_count_mismatch" }; + } + let prevStepHash = `genesis:${args.runId}`; + for (let index = 0; index < args.rows.length; index += 1) { + const row = args.rows[index]; + const stored = args.hashes[index]; + if (String(row.jobId ?? "") !== String(args.jobId ?? "") + || String(row.runId) !== String(args.runId) + || String(row.roomId) !== String(args.roomId) + || row.agentId !== args.agentId) { + return { valid: false, brokenAt: row.idx, reason: "trace_identity_mismatch" }; + } + if (stored.prevStepHash !== prevStepHash) return { valid: false, brokenAt: row.idx, reason: "chain link mismatch" }; + const expected = await sha256hex(canonical(chainCore(row, row, prevStepHash))); + if (expected !== stored.recordHash) return { valid: false, brokenAt: row.idx, reason: "record hash mismatch - tampered" }; + prevStepHash = stored.recordHash; + } + return { valid: true, steps: args.rows.length }; +} diff --git a/convex/agentStepJournal.ts b/convex/agentStepJournal.ts index be26bc48..667c1688 100644 --- a/convex/agentStepJournal.ts +++ b/convex/agentStepJournal.ts @@ -1,13 +1,33 @@ import { v } from "convex/values"; -import { internalMutation, internalQuery } from "./_generated/server"; +import { internalMutation } from "./_generated/server"; -export const get = internalQuery({ +async function requireJournalLease(ctx: any, jobId: any, leaseId?: string) { + const job = await ctx.db.get(jobId); + if (!job) throw new Error("job_not_found"); + const terminal = job.status === "completed" + || job.status === "failed" + || job.status === "blocked" + || job.status === "cancelled"; + if (terminal) throw new Error("job_terminal"); + if (job.leaseId) { + if (!leaseId || job.leaseId !== leaseId || !job.leaseUntil || job.leaseUntil <= Date.now()) { + throw new Error("job_lease_invalid"); + } + } else if (leaseId) { + throw new Error("job_lease_invalid"); + } + return job; +} + +export const get = internalMutation({ args: { jobId: v.id("agentJobs"), + leaseId: v.optional(v.string()), sliceKey: v.string(), step: v.number(), }, - handler: async (ctx, { jobId, sliceKey, step }) => { + handler: async (ctx, { jobId, leaseId, sliceKey, step }) => { + await requireJournalLease(ctx, jobId, leaseId); const row = await ctx.db .query("agentModelStepJournal") .withIndex("by_job_slice_step", (q) => q.eq("jobId", jobId).eq("sliceKey", sliceKey).eq("step", step)) @@ -20,6 +40,7 @@ export const get = internalQuery({ export const record = internalMutation({ args: { jobId: v.id("agentJobs"), + leaseId: v.optional(v.string()), sliceKey: v.string(), step: v.number(), model: v.string(), @@ -28,15 +49,24 @@ export const record = internalMutation({ result: v.any(), }, handler: async (ctx, a) => { + await requireJournalLease(ctx, a.jobId, a.leaseId); const existing = await ctx.db .query("agentModelStepJournal") .withIndex("by_job_slice_step", (q) => q.eq("jobId", a.jobId).eq("sliceKey", a.sliceKey).eq("step", a.step)) .order("asc") .first(); - if (existing) return { id: existing._id, reused: true as const }; + if (existing) { + if ( + existing.model !== a.model + || existing.inputHash !== a.inputHash + || existing.outputHash !== a.outputHash + ) throw new Error("journal_replay_mismatch"); + return { id: existing._id, reused: true as const }; + } const now = Date.now(); const id = await ctx.db.insert("agentModelStepJournal", { jobId: a.jobId, + leaseId: a.leaseId, sliceKey: a.sliceKey, step: a.step, model: a.model, diff --git a/convex/agentStepJournalClient.ts b/convex/agentStepJournalClient.ts index 2051d358..6b4443b8 100644 --- a/convex/agentStepJournalClient.ts +++ b/convex/agentStepJournalClient.ts @@ -4,31 +4,49 @@ import type { AgentStep } from "../src/nodeagent/core/types"; import type { StepJournal } from "../src/nodeagent/core/journal"; import { stableJournalHash } from "../src/nodeagent/core/journal"; -const agentStepJournalGetRef = makeFunctionReference<"query">("agentStepJournal:get") as any; +const agentStepJournalGetRef = makeFunctionReference<"mutation">("agentStepJournal:get") as any; const agentStepJournalRecordRef = makeFunctionReference<"mutation">("agentStepJournal:record") as any; +export type ConvexStepJournal = StepJournal & { + accountingClaims(): Array<{ step: number; outputHash: string; state: "confirmed" | "pending" }>; +}; + export function makeConvexStepJournal(args: { ctx: { - runQuery: (ref: any, args: Record) => Promise; runMutation: (ref: any, args: Record) => Promise; }; jobId: Id<"agentJobs">; + leaseId?: string; sliceKey: string; inputHash?: string; modelName: () => string; -}): StepJournal { +}): ConvexStepJournal { const inputHash = args.inputHash ?? args.sliceKey; + const accounted = new Map(); + const remember = (step: number, result: AgentStep, state: "confirmed" | "pending") => { + accounted.set(step, { + outputHash: stableJournalHash(result), + state, + }); + }; return { async get(step: number) { - return await args.ctx.runQuery(agentStepJournalGetRef, { + const result = await args.ctx.runMutation(agentStepJournalGetRef, { jobId: args.jobId, + leaseId: args.leaseId, sliceKey: args.sliceKey, step, }) as AgentStep | undefined; + if (result) remember(step, result, "confirmed"); + return result; }, async record(step: number, result: AgentStep) { + // The mutation may commit and then lose its response. Track the uncertain claim before + // awaiting so run accounting can reconcile a matching durable row without double charging. + remember(step, result, "pending"); await args.ctx.runMutation(agentStepJournalRecordRef, { jobId: args.jobId, + leaseId: args.leaseId, sliceKey: args.sliceKey, step, model: args.modelName(), @@ -36,6 +54,12 @@ export function makeConvexStepJournal(args: { outputHash: stableJournalHash(result), result, }); + remember(step, result, "confirmed"); + }, + accountingClaims() { + return [...accounted.entries()] + .sort(([left], [right]) => left - right) + .map(([step, value]) => ({ step, outputHash: value.outputHash, state: value.state })); }, }; } diff --git a/convex/agentSteps.ts b/convex/agentSteps.ts index 3ffe25d5..87a2d969 100644 --- a/convex/agentSteps.ts +++ b/convex/agentSteps.ts @@ -1,46 +1,21 @@ /** - * Step-level agent trace — APPEND-ONLY, tamper-evident, the audit + trajectory-eval record. - * Written once per run by the runRoomAgent action; never updated or deleted (corrections - * are new compensating runs, per SEC 17a-4(f) / SOX §802). - * - * - record: insert the run's steps, each chained by SHA-256 (recordHash ← prevStepHash). - * - byRun: the full ordered (tool · args → result · status) sequence — trajectory eval + replay. - * - byElement: every agent write that touched a cell — finance provenance ("why is this value"). - * - verify: re-walk the hash chain — proves no past step was altered. + * Append-only, tamper-evident NodeAgent tool trace. + * Durable-job callers persist this chain atomically with their run accounting. */ import { v } from "convex/values"; import { internalMutation, query } from "./_generated/server"; import { actorProofV, requireActorProof } from "./lib"; - -const statusV = v.union(v.literal("ok"), v.literal("conflict"), v.literal("locked"), v.literal("error")); -const stepV = v.object({ - idx: v.number(), tool: v.string(), args: v.string(), result: v.string(), - status: statusV, ms: v.number(), elementId: v.optional(v.string()), - affectedObjectIds: v.optional(v.array(v.string())), - mutationReceiptIds: v.optional(v.array(v.id("agentMutationReceipts"))), -}); - -async function sha256hex(s: string): Promise { - const buf = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(s)); - return Array.from(new Uint8Array(buf)).map((b) => b.toString(16).padStart(2, "0")).join(""); -} -/** Deterministic, sorted-key serialization (DETERMINISTIC rule) for stable hashing. */ -const canonical = (o: Record) => - JSON.stringify(Object.keys(o).sort().reduce>((acc, k) => { acc[k] = o[k]; return acc; }, {})); -const core = (s: { runId: string; roomId: string; agentId: string; idx: number; tool: string; args: string; result: string; status: string; ms: number; elementId?: string; prevStepHash: string }) => - ({ runId: s.runId, roomId: s.roomId, agentId: s.agentId, idx: s.idx, tool: s.tool, args: s.args, result: s.result, status: s.status, ms: s.ms, elementId: s.elementId ?? "", prevStepHash: s.prevStepHash }); +import { agentTraceStepV, recordAgentStepChain, verifyAgentStepChain } from "./agentStepChain"; export const record = internalMutation({ - args: { jobId: v.optional(v.id("agentJobs")), runId: v.id("agentRuns"), roomId: v.id("rooms"), agentId: v.string(), steps: v.array(stepV) }, - handler: async (ctx, a) => { - const ts = Date.now(); - let prevStepHash = `genesis:${a.runId}`; - for (const s of a.steps) { - const recordHash = await sha256hex(canonical(core({ runId: a.runId, roomId: a.roomId, agentId: a.agentId, ...s, prevStepHash }))); - await ctx.db.insert("agentSteps", { jobId: a.jobId, runId: a.runId, roomId: a.roomId, agentId: a.agentId, idx: s.idx, tool: s.tool, args: s.args, result: s.result, status: s.status, ms: s.ms, elementId: s.elementId, affectedObjectIds: s.affectedObjectIds, mutationReceiptIds: s.mutationReceiptIds, ts, recordHash, prevStepHash }); - prevStepHash = recordHash; - } + args: { + jobId: v.optional(v.id("agentJobs")), + runId: v.id("agentRuns"), + roomId: v.id("rooms"), + agentId: v.string(), + steps: v.array(agentTraceStepV), }, + handler: (ctx, args) => recordAgentStepChain(ctx, args), }); export const byRun = query({ @@ -57,7 +32,10 @@ export const byElement = query({ args: { roomId: v.id("rooms"), elementId: v.string(), requester: actorProofV }, handler: async (ctx, { roomId, elementId, requester }) => { await requireActorProof(ctx, roomId, requester); - return ctx.db.query("agentSteps").withIndex("by_room_element", (q) => q.eq("roomId", roomId).eq("elementId", elementId)).order("desc").collect(); + return ctx.db.query("agentSteps") + .withIndex("by_room_element", (q) => q.eq("roomId", roomId).eq("elementId", elementId)) + .order("desc") + .collect(); }, }); @@ -65,16 +43,20 @@ export const verify = query({ args: { runId: v.id("agentRuns"), requester: actorProofV }, handler: async (ctx, { runId, requester }) => { const run = await ctx.db.get(runId); - if (!run) return { valid: false, steps: 0, reason: "run_not_found" }; + if (!run) return { valid: false as const, steps: 0, reason: "run_not_found" }; await requireActorProof(ctx, run.roomId, requester); - const steps = await ctx.db.query("agentSteps").withIndex("by_run", (q) => q.eq("runId", runId)).collect(); - let prevStepHash = `genesis:${runId}`; - for (const s of steps) { - if (s.prevStepHash !== prevStepHash) return { valid: false, brokenAt: s.idx, reason: "chain link mismatch" }; - const expected = await sha256hex(canonical(core({ ...s, runId: s.runId, roomId: s.roomId, prevStepHash }))); - if (expected !== s.recordHash) return { valid: false, brokenAt: s.idx, reason: "record hash mismatch — tampered" }; - prevStepHash = s.recordHash; - } - return { valid: true, steps: steps.length }; + const steps = await ctx.db.query("agentSteps") + .withIndex("by_run", (q) => q.eq("runId", runId)) + .order("asc") + .collect(); + return verifyAgentStepChain({ + jobId: run.jobId, + runId, + roomId: run.roomId, + agentId: run.agentId, + expectedCount: run.traceRecordCount, + rows: steps, + hashes: steps, + }); }, }); diff --git a/convex/agentWorkflows.ts b/convex/agentWorkflows.ts index 07aae097..088ed59b 100644 --- a/convex/agentWorkflows.ts +++ b/convex/agentWorkflows.ts @@ -35,6 +35,17 @@ export const passiveWorkflow = new WorkflowManager(components.passiveWorkflow, { const MAX_WORKFLOW_SLICES = 200; +function requireSuccessfulSliceResult(result: unknown): void { + if (result && typeof result === "object" && (result as { ok?: unknown }).ok === true) return; + const candidate = result && typeof result === "object" ? result as { reason?: unknown; error?: unknown } : undefined; + const detail = typeof candidate?.reason === "string" + ? candidate.reason + : typeof candidate?.error === "string" + ? candidate.error + : "invalid_result"; + throw new Error(`agent_slice_failed:${detail.slice(0, 1_000)}`); +} + export const freeAutoWorkflow = workflow.define({ args: { jobId: v.id("agentJobs") }, returns: v.null(), @@ -47,7 +58,7 @@ export const freeAutoWorkflow = workflow.define({ if (before.terminal) return null; const delayMs = Math.max(0, (before.nextRunAt ?? before.now) - before.now); if (delayMs > 0) await step.sleep(delayMs, { name: `free-auto-delay-${slice}` }); - await step.runAction(internal.agentJobRunner.runFreeAutoJobSlice, { jobId }, { name: `free-auto-slice-${slice + 1}`, retry: false }); + requireSuccessfulSliceResult(await step.runAction(internal.agentJobRunner.runFreeAutoJobSlice, { jobId }, { name: `free-auto-slice-${slice + 1}`, retry: false })); return null; }); @@ -61,7 +72,7 @@ export const passiveRoomWorkWorkflow = passiveWorkflow.define({ if (before.terminal) return null; const delayMs = Math.max(0, (before.nextRunAt ?? before.now) - before.now); if (delayMs > 0) await step.sleep(delayMs, { name: `passive-delay-${slice}` }); - await step.runAction(internal.agentJobRunner.runFreeAutoJobSlice, { jobId }, { name: `passive-slice-${slice + 1}`, retry: false }); + requireSuccessfulSliceResult(await step.runAction(internal.agentJobRunner.runFreeAutoJobSlice, { jobId }, { name: `passive-slice-${slice + 1}`, retry: false })); return null; }); diff --git a/convex/schema.ts b/convex/schema.ts index 33f62140..866cd6a2 100644 --- a/convex/schema.ts +++ b/convex/schema.ts @@ -852,6 +852,7 @@ export default defineSchema({ model: v.string(), goal: v.string(), steps: v.number(), + traceRecordCount: v.optional(v.number()), modelCalls: v.optional(v.number()), toolCalls: v.number(), conflictsSurvived: v.number(), @@ -869,7 +870,10 @@ export default defineSchema({ handoff: v.optional(v.any()), idempotencyKey: v.optional(v.string()), createdAt: v.number(), - }).index("by_room", ["roomId", "createdAt"]).index("by_idempotency", ["idempotencyKey", "createdAt"]), + }) + .index("by_room", ["roomId", "createdAt"]) + .index("by_job", ["jobId", "createdAt"]) + .index("by_idempotency", ["idempotencyKey", "createdAt"]), // ── Credit ledger (pilot wallet). The credit math + caps live in // src/nodeagent/core/creditModel.ts (the single source of truth, imported here). @@ -1009,6 +1013,9 @@ export default defineSchema({ agentModelStepJournal: defineTable({ jobId: v.id("agentJobs"), + leaseId: v.optional(v.string()), + accountedRunId: v.optional(v.id("agentRuns")), + accountingClaimedAt: v.optional(v.number()), sliceKey: v.string(), step: v.number(), model: v.string(), diff --git a/src/app/store.tsx b/src/app/store.tsx index 727e8e9d..066a5178 100644 --- a/src/app/store.tsx +++ b/src/app/store.tsx @@ -50,7 +50,12 @@ export type EditFeedback = { ok: boolean; reason?: string; version?: number }; type UndoEntry = { roomId: string; op: ChangeOp }; export type AgentCostKind = "exact" | "estimated"; export type AgentRunTelemetry = { model: string; steps: number; toolCalls: number; inputTokens: number; outputTokens: number; costUsd: number; costKind: AgentCostKind; ms: number }; -type PersistedAgentRunTelemetry = Omit & { costKind?: AgentCostKind }; +type PersistedAgentRunTelemetry = Omit & { + _id?: unknown; + id?: unknown; + jobId?: unknown; + costKind?: AgentCostKind; +}; function persistedCostKind(costKind: AgentCostKind | undefined): AgentCostKind { return costKind ?? "estimated"; @@ -99,6 +104,55 @@ export type AgentJobTelemetry = { updatedAt: number; }; +type AgentJobRunDetailCorrelation = { + job?: { _id?: unknown; id?: unknown; latestRunId?: unknown }; + latestRun?: PersistedAgentRunTelemetry | null; +}; + +function correlationId(value: unknown): string | undefined { + if (value === undefined || value === null) return undefined; + const id = String(value).trim(); + return id || undefined; +} + +function persistedRunId(run: PersistedAgentRunTelemetry): string | undefined { + return correlationId(run._id) ?? correlationId(run.id); +} + +/** Resolve only telemetry proven to belong to the selected durable job. Room run lists are + * newest-first but room-wide, while detail.latestRun is selected-job-specific and can recover a + * run that fell outside the bounded room list. */ +export function selectAgentRunTelemetryForJob( + runs: readonly PersistedAgentRunTelemetry[], + job: Pick, + detail?: AgentJobRunDetailCorrelation | null, +): AgentRunTelemetry | null { + const jobId = correlationId(job.id); + if (!jobId) return null; + + const detailJobId = correlationId(detail?.job?._id) ?? correlationId(detail?.job?.id); + const detailMatchesJob = detailJobId === jobId; + const latestRunId = correlationId(job.latestRunId) + ?? (detailMatchesJob ? correlationId(detail?.job?.latestRunId) : undefined); + const detailRun = detail?.latestRun ?? undefined; + + if (latestRunId) { + const exactCandidates = detailRun ? [detailRun, ...runs] : runs; + const exactRun = exactCandidates.find((run) => { + if (persistedRunId(run) !== latestRunId) return false; + const runJobId = correlationId(run.jobId); + return runJobId === undefined || runJobId === jobId; + }); + return exactRun ? projectAgentRunTelemetry(exactRun) : null; + } + + if (detailMatchesJob && detailRun && correlationId(detailRun.jobId) === jobId) { + return projectAgentRunTelemetry(detailRun); + } + const newestJobRun = runs.find((run) => correlationId(run.jobId) === jobId); + return newestJobRun ? projectAgentRunTelemetry(newestJobRun) : null; +} + /** Shape of a free-auto agent job row from the convex jobs subscription (used by lastLongFreeJob + activeLongFreeJobs). */ type FreeJobRow = { _id: string; goal?: string; status: string; entrypoint?: string; scope?: string; runtime?: string; attempts: number; maxAttempts: number; @@ -373,7 +427,7 @@ export interface RoomStore { startLongFreeAgent(input: AgentAskInput): Promise; /** Enrich every PENDING company on the research sheet (ParselyFi loop) — status-gated, sourced. */ askResearch(): Promise; - /** The most recent agent run's telemetry (model · tokens · cost · latency), or null. */ + /** The selected durable job's correlated run telemetry, or the room's latest run when no durable job exists. */ lastRun(): AgentRunTelemetry | null; lastLongFreeJob(): AgentJobTelemetry | null; lastLongFreeJobAttempts(): AgentJobAttemptTelemetry[]; @@ -2237,8 +2291,16 @@ export function ConvexStoreProvider({ roomId, me, proof, children }: { roomId: s }); }, lastRun: () => { - const r = (runs as unknown as PersistedAgentRunTelemetry[])[0]; - return r ? projectAgentRunTelemetry(r) : null; + const roomRuns = runs as unknown as PersistedAgentRunTelemetry[]; + if (!selectedLongJobRow) { + const roomRun = roomRuns[0]; + return roomRun ? projectAgentRunTelemetry(roomRun) : null; + } + return selectAgentRunTelemetryForJob( + roomRuns, + { id: String(selectedLongJobRow._id), latestRunId: selectedLongJobRow.latestRunId ? String(selectedLongJobRow.latestRunId) : undefined }, + jobDetail as unknown as AgentJobRunDetailCorrelation | null | undefined, + ); }, lastLongFreeJob: () => { const j = selectedLongJobRow; diff --git a/src/nodeagent/core/runtime.ts b/src/nodeagent/core/runtime.ts index 9cf9dcb5..2ab9499c 100644 --- a/src/nodeagent/core/runtime.ts +++ b/src/nodeagent/core/runtime.ts @@ -84,6 +84,14 @@ function providerFailureUsage(error: unknown): TokenUsage | undefined { }; } +function providerSpendBudgetBlocked(error: unknown): boolean { + if (!error || typeof error !== "object") return false; + const receipt = (error as { receipt?: unknown }).receipt; + return !!receipt + && typeof receipt === "object" + && (receipt as { stopReason?: unknown }).stopReason === "spend_budget"; +} + function requiredNoToolMissCount(messages: AgentMessage[]): number { let max = 0; const pattern = new RegExp(`${TOOL_REQUIRED_NO_CALL_MARKER}\\s+(\\d+)\\/${TOOL_REQUIRED_NO_CALL_TERMINAL_AFTER}`, "i"); @@ -905,10 +913,17 @@ export async function runAgent(opts: { signal: signal.signal, onTextDelta: opts.onTextDelta ? (text) => opts.onTextDelta?.(text, step) : undefined, toolChoice: requiresToolThisTurn ? "required" : "auto", + maxCostUsd: opts.spendLimits?.maxCostUsd === undefined + ? undefined + : Math.max(0, opts.spendLimits.maxCostUsd - costUsd), }), signal.signal); } catch (error) { const failedUsage = providerFailureUsage(error); if (failedUsage) recordModelUsage(failedUsage); + if (providerSpendBudgetBlocked(error)) { + const handoff = emitHandoff(step, "spend_budget", attemptedSteps); + return finish("spend_budget", attemptedSteps, true, handoff); + } if (signal.signal?.aborted || (shouldHandoffForTime() && isAbortLike(error))) { const handoff = emitHandoff(step, "time_budget", attemptedSteps); return finish("time_budget", attemptedSteps, true, handoff); @@ -917,10 +932,11 @@ export async function runAgent(opts: { } finally { signal.cancel(); } - await opts.journal?.record(step, fresh); - // Count/bill only real provider calls. A failover adapter can represent several provider - // requests in one logical model.next; replayed journal steps remain excluded. + // Capture provider usage before the durable journal write. A lease can expire after the + // response arrives but before persistence; that call still happened and must survive in + // the partial receipt even when the fenced journal rejects the write. recordModelUsage(fresh.usage); + await opts.journal?.record(step, fresh); out = fresh; } if (out.text) finalText = out.text; diff --git a/src/nodeagent/core/types.ts b/src/nodeagent/core/types.ts index 300277d6..0e251014 100644 --- a/src/nodeagent/core/types.ts +++ b/src/nodeagent/core/types.ts @@ -80,6 +80,8 @@ export interface AgentModel { onTextDelta?: (text: string) => void | Promise; /** Hint for providers that support OpenAI-style tool_choice. Runtime still validates writes. */ toolChoice?: AgentToolChoice; + /** Remaining USD available to this logical turn, including provider failover attempts. */ + maxCostUsd?: number; }): Promise; } diff --git a/src/nodeagent/models/convexModel.ts b/src/nodeagent/models/convexModel.ts index 125523b8..d49fa961 100644 --- a/src/nodeagent/models/convexModel.ts +++ b/src/nodeagent/models/convexModel.ts @@ -8,7 +8,7 @@ */ import type { AgentMessage, AgentModel, AgentModelRouteState, AgentStep, AgentTool, AgentToolChoice, TokenUsage, ToolCall } from "../core/types"; -import { getModelPricing, getProviderForModel, resolveModelAlias } from "./modelCatalog"; +import { getModelPricing, getProviderForModel, modelPricing, resolveModelAlias } from "./modelCatalog"; import { isOpenRouterFreeAutoModel, openRouterFreeCandidateTimeoutMs, @@ -110,6 +110,10 @@ type GeminiResponse = { const OPENROUTER_REFERER = "https://noderoom.local"; const OPENROUTER_TITLE = "NodeRoom benchmark"; const DEFAULT_MAX_OUTPUT_TOKENS = 8_192; +const UNKNOWN_MODEL_RESERVE_INPUT_PER_1M = Object.values(modelPricing) + .reduce((highest, pricing) => Math.max(highest, pricing.inputPer1M), 1); +const UNKNOWN_MODEL_RESERVE_OUTPUT_PER_1M = Object.values(modelPricing) + .reduce((highest, pricing) => Math.max(highest, pricing.outputPer1M), 5); const TRANSIENT_RE = /(\b429\b|\b5\d\d\b|rate.?limit|overloaded|temporar|timed?.?out|ECONNRESET|ETIMEDOUT|EAI_AGAIN|ENOTFOUND|fetch failed|socket hang up|service unavailable)/i; export type ConvexModelOptions = { @@ -136,33 +140,50 @@ class ProviderStepError extends Error { } } -function openAiUsage(usage?: OpenAiChatResponse["usage"]): TokenUsage { +function providerTokenCount(value: unknown): number | undefined { + return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : undefined; +} + +function openAiUsage(usage?: OpenAiChatResponse["usage"]): TokenUsage | undefined { + if (!usage) return undefined; + const inputTokens = providerTokenCount(usage.prompt_tokens ?? usage.input_tokens); + const outputTokens = providerTokenCount(usage.completion_tokens ?? usage.output_tokens); + const rawCachedInputTokens = usage.prompt_tokens_details?.cached_tokens + ?? usage.input_tokens_details?.cached_tokens; + const cachedInputTokens = rawCachedInputTokens === undefined ? 0 : providerTokenCount(rawCachedInputTokens); + if (inputTokens === undefined || outputTokens === undefined || cachedInputTokens === undefined) return undefined; return { - inputTokens: usage?.prompt_tokens ?? usage?.input_tokens ?? 0, - outputTokens: usage?.completion_tokens ?? usage?.output_tokens ?? 0, - cachedInputTokens: usage?.prompt_tokens_details?.cached_tokens - ?? usage?.input_tokens_details?.cached_tokens - ?? 0, + inputTokens, + outputTokens, + cachedInputTokens, }; } -function anthropicUsage(usage?: AnthropicResponse["usage"]): TokenUsage { - const uncached = usage?.input_tokens ?? 0; - const cached = usage?.cache_read_input_tokens ?? 0; - const cacheCreation = usage?.cache_creation_input_tokens ?? 0; +function anthropicUsage(usage?: AnthropicResponse["usage"]): TokenUsage | undefined { + if (!usage) return undefined; + const uncached = providerTokenCount(usage.input_tokens); + const outputTokens = providerTokenCount(usage.output_tokens); + const cached = usage.cache_read_input_tokens === undefined ? 0 : providerTokenCount(usage.cache_read_input_tokens); + const cacheCreation = usage.cache_creation_input_tokens === undefined ? 0 : providerTokenCount(usage.cache_creation_input_tokens); + if (uncached === undefined || outputTokens === undefined || cached === undefined || cacheCreation === undefined) return undefined; return { inputTokens: uncached + cached + cacheCreation, - outputTokens: usage?.output_tokens ?? 0, + outputTokens, cachedInputTokens: cached, cacheCreationInputTokens: cacheCreation, }; } -function geminiUsage(usage?: GeminiResponse["usageMetadata"]): TokenUsage { +function geminiUsage(usage?: GeminiResponse["usageMetadata"]): TokenUsage | undefined { + if (!usage) return undefined; + const inputTokens = providerTokenCount(usage.promptTokenCount); + const outputTokens = providerTokenCount(usage.candidatesTokenCount); + const cachedInputTokens = usage.cachedContentTokenCount === undefined ? 0 : providerTokenCount(usage.cachedContentTokenCount); + if (inputTokens === undefined || outputTokens === undefined || cachedInputTokens === undefined) return undefined; return { - inputTokens: usage?.promptTokenCount ?? 0, - outputTokens: usage?.candidatesTokenCount ?? 0, - cachedInputTokens: usage?.cachedContentTokenCount ?? 0, + inputTokens, + outputTokens, + cachedInputTokens, }; } @@ -182,11 +203,11 @@ export function convexModel(modelId: string, options: ConvexModelOptions = {}): routeState() { return snapshotConcreteRouteState(concreteRouteState); }, - async next({ system, messages, tools, signal, onTextDelta, toolChoice }) { + async next({ system, messages, tools, signal, onTextDelta, toolChoice, maxCostUsd }) { // Gateway PII firewall — redact PII/secrets from the system + user content before the prompt leaves. const safeSystem = redactPII(system).text; const safeMessages = messages.map((m) => (m.role === "user" && m.content ? { ...m, content: redactPII(m.content).text } : m)); - const { step, resolvedModel } = await generateConvexAgentStep(aliasModelId, safeSystem, safeMessages, tools, entrypoint, signal, onTextDelta, toolChoice, freeAutoMode, fallbackModelIds, concreteRouteState); + const { step, resolvedModel } = await generateConvexAgentStep(aliasModelId, safeSystem, safeMessages, tools, entrypoint, signal, onTextDelta, toolChoice, freeAutoMode, fallbackModelIds, concreteRouteState, maxCostUsd); resolvedModelId = resolvedModel; return step; }, @@ -216,6 +237,44 @@ function exactConvexUsageCost( ) / 1_000_000; } +function conservativeRequestInputTokenUpperBound( + system: string, + messages: AgentMessage[], + tools: AgentTool[], +): number { + const serialized = JSON.stringify({ + system, + messages, + tools: tools.map((tool) => ({ + name: tool.name, + description: tool.description, + parameters: toolParameters(tool.name), + })), + }); + const utf8Bytes = new TextEncoder().encode(serialized).byteLength; + const providerFramingAllowance = 64 * (messages.length + tools.length + 2); + return utf8Bytes + providerFramingAllowance; +} + +function convexAttemptCostReservationUsd( + modelId: string, + inputTokenUpperBound: number, + maxCostUsd?: number, +): number { + const pricing = getModelPricing(resolveModelAlias(modelId)); + if (!pricing && Number.isFinite(maxCostUsd)) { + const budget = Math.max(0, maxCostUsd ?? 0); + return budget + Math.max(1e-9, Math.max(1, budget) * 1e-9); + } + const inputPer1M = pricing?.inputPer1M ?? UNKNOWN_MODEL_RESERVE_INPUT_PER_1M; + const outputPer1M = pricing?.outputPer1M ?? UNKNOWN_MODEL_RESERVE_OUTPUT_PER_1M; + if (inputPer1M === 0 && outputPer1M === 0) return 0; + return ( + inputTokenUpperBound * inputPer1M + + modelMaxOutputTokens() * outputPer1M + ) / 1_000_000; +} + async function generateConvexAgentStep( modelId: string, system: string, @@ -228,6 +287,7 @@ async function generateConvexAgentStep( freeAutoMode?: OpenRouterFreeModelMode, fallbackModelIds: string[] = [], concreteRouteState?: ConcreteRouteState, + maxCostUsd?: number, ) { assertProviderRouteAllowed({ model: modelId, entrypoint, env: process.env }); let providerRequests = 0; @@ -254,6 +314,7 @@ async function generateConvexAgentStep( maxAttempts: Math.min(candidates.length, openRouterFreeAutoLimit()), deadlineAt: requestStartedAt + openRouterFreeRequestTimeoutMs(), reserveMs: openRouterFreeRequestReserveMs(), + maxCostUsd, }, attemptTimeoutMs: openRouterFreeCandidateTimeoutMs(), signal, @@ -355,30 +416,38 @@ async function generateConvexAgentStep( const state = concreteRouteState ?? { preferredModelId: modelId, cooldownUntil: new Map() }; const candidateIds = orderedConcreteCandidateIds(modelId, fallbackModelIds, state.preferredModelId); + const inputTokenUpperBound = conservativeRequestInputTokenUpperBound(system, messages, tools); const candidates = candidateIds.map((id) => ({ id, provider: getProviderForModel(id) ?? undefined, cooldownUntil: state.cooldownUntil.get(id), + estimatedCostUsd: convexAttemptCostReservationUsd(id, inputTokenUpperBound, maxCostUsd), })); + const concreteProviderRequestAttempts = new Set(); let aggregateInputTokens = 0; let aggregateOutputTokens = 0; let aggregateCachedInputTokens = 0; let aggregateCacheCreationInputTokens = 0; let aggregateCostKnown = true; + let aggregateUsedCostReservation = false; const candidateText = acceptedCandidateTextBuffer(onTextDelta); const failover = await runQualityFailover({ candidates, - budget: { maxAttempts: candidateIds.length }, + budget: { maxAttempts: candidateIds.length, maxCostUsd }, attemptTimeoutMs: concreteCandidateTimeoutMs(), signal, execute: async (candidate, context) => { const providerRoute = assertProviderRouteAllowed({ model: candidate.id, entrypoint, env: process.env }); const requestsBefore = providerRequests; + const onCandidateProviderRequest = () => { + concreteProviderRequestAttempts.add(context.attempt); + onProviderRequest(); + }; let step: AgentStep; try { step = withProviderRoute( await withRetry( - () => providerStep(candidate.id, system, messages, tools, context.signal, candidateText.sink(candidate.id), toolChoice, onProviderRequest), + () => providerStep(candidate.id, system, messages, tools, context.signal, candidateText.sink(candidate.id), toolChoice, onCandidateProviderRequest), context.signal, 0, ), @@ -423,12 +492,21 @@ async function generateConvexAgentStep( await candidateText.settle(candidate.id, attempt.outcome === "accepted"); updateConcreteRouteState(state, candidate.id, attempt.outcome, attempt.providerFailureCategory); }, - measureCostUsd: ({ candidate, result, error }) => { + measureCostUsd: ({ candidate, attempt, result, error }) => { + if (!concreteProviderRequestAttempts.has(attempt)) return 0; const usage = result?.usage ?? errorTokenUsage(error); - if (!usage) return 0; + if (!usage) { + aggregateCostKnown = false; + aggregateUsedCostReservation = true; + return undefined; + } const measured = exactConvexUsageCost(candidate.id, usage); - if (measured === undefined) aggregateCostKnown = false; - return measured ?? 0; + if (measured === undefined) { + aggregateCostKnown = false; + aggregateUsedCostReservation = true; + return undefined; + } + return measured; }, }); if (providerRequests > 0 && failover.receipt.routeAttempts.some((attempt) => attempt.reason === "candidate_timeout")) { @@ -455,7 +533,11 @@ async function generateConvexAgentStep( cachedInputTokens: aggregateCachedInputTokens, cacheCreationInputTokens: aggregateCacheCreationInputTokens, modelCalls: providerRequests, - ...(aggregateCostKnown ? { costUsd: failover.receipt.budget.spentCostUsd, costKind: "exact" as const } : {}), + ...(aggregateCostKnown + ? { costUsd: failover.receipt.budget.spentCostUsd, costKind: "exact" as const } + : aggregateUsedCostReservation + ? { costUsd: failover.receipt.budget.spentCostUsd, costKind: "estimated" as const } + : {}), }, providerRoute: { ...(failover.result.providerRoute as ProviderRouteReceipt), @@ -770,17 +852,12 @@ async function openAiCompatibleBlockingStep(args: { tool: tc.function?.name ?? "unknown_tool", args: parseJsonObject(tc.function?.arguments ?? "{}"), })); + const usage = openAiUsage(res.usage); return { text: message.content || undefined, toolCalls, done: toolCalls.length === 0, - usage: { - inputTokens: res.usage?.prompt_tokens ?? res.usage?.input_tokens ?? 0, - outputTokens: res.usage?.completion_tokens ?? res.usage?.output_tokens ?? 0, - cachedInputTokens: res.usage?.prompt_tokens_details?.cached_tokens - ?? res.usage?.input_tokens_details?.cached_tokens - ?? 0, - }, + ...(usage ? { usage } : {}), }; } @@ -862,11 +939,12 @@ async function openAiCompatibleStreamStep(args: { tool: tc.name ?? "unknown_tool", args: parseOpenAiStreamToolArgs(tc.name, tc.argsText), })); + const tokenUsage = openAiUsage(usage); return { text: text || undefined, toolCalls, done: toolCalls.length === 0, - usage: openAiUsage(usage), + ...(tokenUsage ? { usage: tokenUsage } : {}), }; } @@ -901,11 +979,12 @@ async function anthropicStep( tool: p.name ?? "unknown_tool", args: p.input ?? {}, })); + const usage = anthropicUsage(res.usage); return { text: text || undefined, toolCalls, done: toolCalls.length === 0, - usage: anthropicUsage(res.usage), + ...(usage ? { usage } : {}), }; } @@ -938,15 +1017,12 @@ async function geminiStep( args: p.functionCall.args ?? {}, providerMetadata: p.thoughtSignature || p.thought_signature ? { geminiThoughtSignature: p.thoughtSignature ?? p.thought_signature } : undefined, })); + const usage = geminiUsage(res.usageMetadata); return { text: text || undefined, toolCalls, done: toolCalls.length === 0, - usage: { - inputTokens: res.usageMetadata?.promptTokenCount ?? 0, - outputTokens: res.usageMetadata?.candidatesTokenCount ?? 0, - cachedInputTokens: res.usageMetadata?.cachedContentTokenCount ?? 0, - }, + ...(usage ? { usage } : {}), }; } @@ -1008,11 +1084,12 @@ async function geminiStreamStep( throw new ProviderStepError(error, usage ? geminiUsage(usage) : undefined); } + const tokenUsage = geminiUsage(usage); return { text: text || undefined, toolCalls, done: toolCalls.length === 0, - usage: geminiUsage(usage), + ...(tokenUsage ? { usage: tokenUsage } : {}), }; } diff --git a/src/nodeagent/models/qualityFailover.ts b/src/nodeagent/models/qualityFailover.ts index f05c952e..25c85aa5 100644 --- a/src/nodeagent/models/qualityFailover.ts +++ b/src/nodeagent/models/qualityFailover.ts @@ -202,9 +202,10 @@ export interface QualityFailoverOptions< candidate: TCandidate, ) => ProviderFailureClassification; estimateCostUsd?: (candidate: TCandidate) => number; + /** Return undefined when provider usage is unavailable so the pre-call reservation remains charged. */ measureCostUsd?: ( context: QualityFailoverCostContext, - ) => number; + ) => number | undefined; /** Receives provider and task-quality outcomes separately; use it to update existing health ledgers. */ onRouteAttempt?: ( attempt: Readonly, @@ -864,9 +865,10 @@ function measureCost( options: QualityFailoverOptions, context: QualityFailoverCostContext, ): number { - const measured = options.measureCostUsd?.(context) ?? context.estimatedCostUsd; - assertNonNegativeFinite(measured, `measured cost for ${context.candidate.id}`); - return measured; + const measured = options.measureCostUsd?.(context); + const settled = measured === undefined ? context.estimatedCostUsd : measured; + assertNonNegativeFinite(settled, `measured cost for ${context.candidate.id}`); + return settled; } function classifyProviderFailure( diff --git a/src/ui/RoomShell.tsx b/src/ui/RoomShell.tsx index 1cd43797..64c2cb59 100644 --- a/src/ui/RoomShell.tsx +++ b/src/ui/RoomShell.tsx @@ -7,7 +7,7 @@ import { useEffect, useRef, useState, type CSSProperties, type ReactNode } from "react"; import { PanelLeft, Table2, PanelRight, Moon, Sun, LogOut, ShieldCheck, X, HelpCircle, Copy, Check, MessageCircle, Sparkles, SlidersHorizontal, Palette, Gauge, Play, ChevronLeft, ChevronRight, Crosshair, WifiOff } from "lucide-react"; -import { useStore, type AgentCostKind, type ActorProof } from "../app/store"; +import { useStore, type AgentCostKind, type AgentJobTelemetry, type AgentRunTelemetry, type ActorProof } from "../app/store"; import { OFFLINE_QUEUE_MAX } from "../notifications/offlineQueue"; import { Chat } from "./Chat"; import { Artifact } from "./panels/Artifact"; @@ -47,6 +47,20 @@ export function formatAgentCost(costUsd: number, costKind?: AgentCostKind): stri return `${costKind === "exact" ? "" : "≈"}$${costUsd.toFixed(3)}`; } +export function selectedJobSignalTelemetry(job: AgentJobTelemetry, run: AgentRunTelemetry | null): { evalValue: string; costValue: string } { + if (run) { + return { + evalValue: `${run.model} | ${run.toolCalls} tools`, + costValue: formatAgentCost(run.costUsd, run.costKind), + }; + } + const toolTotal = job.toolCallCount === undefined ? "" : ` | ${job.toolCallCount} tools total`; + return { + evalValue: `route ${job.modelPolicy || "not reported"}${toolTotal}`, + costValue: job.costUsd === undefined ? "not reported" : `${formatAgentCost(job.costUsd, job.costKind)} job total`, + }; +} + export function preferredRoomArtifact(arts: T[]): T | undefined { const scaleResearch = arts.find((a) => a.kind === "sheet" && @@ -993,6 +1007,7 @@ function SignalStatusStrip({ const jobStatus = job?.status ?? ""; const jobRisk = ["failed", "blocked", "cancelled", "paused"].includes(jobStatus); const jobLive = !!job && !["completed", "failed", "cancelled", "blocked", "paused"].includes(jobStatus); + const jobTelemetry = job ? selectedJobSignalTelemetry(job, run) : null; const credit = store.creditBalance?.(); const demoCreditValue = q3MemoryDemo && credit?.demo ? 18 : credit?.availableCredits; const reconciledStatusPrefix = "Room NodeAgent · "; @@ -1009,11 +1024,11 @@ function SignalStatusStrip({ : []), ...(proposals.length ? [{ k: "Review", v: `${proposals.length} pending` }] : []), ...(jobRisk ? [{ k: "Run", v: jobStatus }] : []), - ...(jobLive + ...(jobLive && jobTelemetry ? [ { k: "Agents", v: `${sessions.length} active` }, - { k: "Eval", v: run ? `${run.model} | ${run.toolCalls} tools` : "running" }, - { k: "Cost", v: run ? formatAgentCost(run.costUsd, run.costKind) : job ? job.modelPolicy : "-" }, + { k: "Eval", v: jobTelemetry.evalValue }, + { k: "Cost", v: jobTelemetry.costValue }, ] : []), ]; diff --git a/tests/agentJobsRuntime.test.ts b/tests/agentJobsRuntime.test.ts index 11a4e597..674fee97 100644 --- a/tests/agentJobsRuntime.test.ts +++ b/tests/agentJobsRuntime.test.ts @@ -18,7 +18,7 @@ delete (modules as Record)["../convex/agent.ts"]; // Stub their scheduled entrypoints so scheduler.runAfter(internal..*, ...) resolves to a // no-op instead of throwing "Could not find module" as an unhandled rejection after assertions pass. (modules as Record)["../convex/agentJobRunner.ts"] = async () => ({ - runFreeAutoJobSlice: internalAction({ args: { jobId: v.id("agentJobs") }, handler: async () => null }), + runFreeAutoJobSlice: internalAction({ args: { jobId: v.id("agentJobs") }, handler: async () => ({ ok: true as const }) }), }); (modules as Record)["../convex/embeddingRunner.ts"] = async () => ({ runOne: internalAction({ args: {}, handler: async () => null }), @@ -1347,8 +1347,8 @@ describe("agentJobs runtime contract", () => { outputHash: "out-a", result, }); - const replay = await t.query(internal.agentStepJournal.get, { jobId, sliceKey: "slice-a", step: 0 }); - const second = await t.mutation(internal.agentStepJournal.record, { + const replay = await t.mutation(internal.agentStepJournal.get, { jobId, sliceKey: "slice-a", step: 0 }); + await expect(t.mutation(internal.agentStepJournal.record, { jobId, sliceKey: "slice-a", step: 0, @@ -1356,18 +1356,91 @@ describe("agentJobs runtime contract", () => { inputHash: "slice-a", outputHash: "out-b", result: { text: "overwritten", toolCalls: [], done: true }, - }); - const replayAfterDuplicate = await t.query(internal.agentStepJournal.get, { jobId, sliceKey: "slice-a", step: 0 }); + })).rejects.toThrow("journal_replay_mismatch"); + const replayAfterDuplicate = await t.mutation(internal.agentStepJournal.get, { jobId, sliceKey: "slice-a", step: 0 }); const detail = await t.query(api.agentJobs.detail, { jobId, requester: proof }); expect(first.reused).toBe(false); - expect(second.reused).toBe(true); expect(replay).toEqual(result); expect(replayAfterDuplicate).toEqual(result); expect(detail?.modelJournal).toHaveLength(1); expect(detail?.modelJournal[0].outputHash).toBe("out-a"); }); + it("fences model-step journal access by the current live lease while allowing safe replay after reclaim", async () => { + const { t, proof, roomId, artifactId } = await setupRoom(); + const { jobId } = await t.mutation(api.agentJobs.createOrReuse, jobArgs({ + roomId, + artifactId, + proof, + idempotencyKey: "job-runtime-journal-lease", + })); + await t.mutation(internal.agentJobs.claimSlice, { jobId, leaseId: "lease-journal-1", leaseMs: 60_000 }); + const result = { + text: "inspected", + toolCalls: [], + done: false, + usage: { inputTokens: 12, outputTokens: 3, modelCalls: 1 }, + }; + + await expect(t.mutation(internal.agentStepJournal.record, { + jobId, + sliceKey: "slice-leased", + step: 0, + model: "gemini-3.5-flash", + inputHash: "slice-leased", + outputHash: "out-leased", + result, + })).rejects.toThrow("job_lease_invalid"); + const first = await t.mutation(internal.agentStepJournal.record, { + jobId, + leaseId: "lease-journal-1", + sliceKey: "slice-leased", + step: 0, + model: "gemini-3.5-flash", + inputHash: "slice-leased", + outputHash: "out-leased", + result, + }); + await t.run((ctx) => ctx.db.patch(jobId, { leaseUntil: Date.now() - 1 })); + await expect(t.mutation(internal.agentStepJournal.get, { + jobId, + leaseId: "lease-journal-1", + sliceKey: "slice-leased", + step: 0, + })).rejects.toThrow("job_lease_invalid"); + await expect(t.mutation(internal.agentStepJournal.record, { + jobId, + leaseId: "lease-journal-1", + sliceKey: "slice-leased", + step: 1, + model: "gemini-3.5-flash", + inputHash: "slice-leased", + outputHash: "late-output", + result, + })).rejects.toThrow("job_lease_invalid"); + + const reclaimed = await t.mutation(internal.agentJobs.claimSlice, { jobId, leaseId: "lease-journal-2", leaseMs: 60_000 }); + const replay = await t.mutation(internal.agentStepJournal.get, { + jobId, + leaseId: "lease-journal-2", + sliceKey: "slice-leased", + step: 0, + }); + await expect(t.mutation(internal.agentStepJournal.get, { + jobId, + leaseId: "lease-journal-1", + sliceKey: "slice-leased", + step: 0, + })).rejects.toThrow("job_lease_invalid"); + + expect(first.reused).toBe(false); + expect(reclaimed?.attempt).toBe(2); + expect(replay).toEqual(result); + const detail = await t.query(api.agentJobs.detail, { jobId, requester: proof }); + expect(detail?.modelJournal[0]).toMatchObject({ leaseId: "lease-journal-1", outputHash: "out-leased" }); + }); + it("claimSlice creates an active lease and finishSlice releases it only for the matching lease", async () => { const { t, proof, roomId, artifactId } = await setupRoom(); const { jobId } = await t.mutation(api.agentJobs.createOrReuse, jobArgs({ roomId, artifactId, proof, idempotencyKey: "job-runtime-lease" })); @@ -1416,6 +1489,122 @@ describe("agentJobs runtime contract", () => { expect(detail?.reasoningFrames.map((frame) => frame.status)).toEqual(["completed", "completed", "completed"]); }); + it("rejects finishSlice after the exact lease deadline even before the watchdog runs", async () => { + const { t, proof, roomId, artifactId } = await setupRoom(); + const { jobId } = await t.mutation(api.agentJobs.createOrReuse, jobArgs({ + roomId, + artifactId, + proof, + idempotencyKey: "job-runtime-expired-finish-fence", + })); + await t.mutation(internal.agentJobs.claimSlice, { jobId, leaseId: "lease-expired-finish", leaseMs: 60_000 }); + await t.run((ctx) => ctx.db.patch(jobId, { leaseUntil: Date.now() - 1 })); + + const finished = await t.mutation(internal.agentJobs.finishSlice, finishSliceArgs({ + jobId, + leaseId: "lease-expired-finish", + attempt: 1, + })); + const detail = await t.query(api.agentJobs.detail, { jobId, requester: proof }); + + expect(finished).toEqual({ ok: false, reason: "lease_expired" }); + expect(detail?.job.status).toBe("running"); + expect(detail?.attempts).toHaveLength(0); + }); + + it("terminalizes a current workflow failure instead of swallowing it after multiple attempts", async () => { + const { t, proof, roomId, artifactId } = await setupRoom(); + const failed = await t.mutation(api.agentJobs.createOrReuse, jobArgs({ + roomId, + artifactId, + proof, + idempotencyKey: "job-runtime-workflow-result-failure", + })); + await t.run((ctx) => ctx.db.patch(failed.jobId, { + status: "running" as const, + attempts: 2, + workflowId: "workflow-current", + leaseId: "lease-current", + leaseUntil: Date.now() + 60_000, + })); + + const completed = await t.mutation(internal.agentJobs.recordWorkflowComplete, { + jobId: failed.jobId, + workflowId: "workflow-current", + resultKind: "failed" as const, + error: "agent_slice_failed:agent_job_finish_rejected:lease_mismatch", + }); + const failedDetail = await t.query(api.agentJobs.detail, { jobId: failed.jobId, requester: proof }); + + expect(completed).toEqual({ ok: true, terminal: true }); + expect(failedDetail?.job).toMatchObject({ + status: "failed", + error: "agent_slice_failed:agent_job_finish_rejected:lease_mismatch", + leaseId: "", + leaseUntil: 0, + }); + + const concurrent = await t.mutation(api.agentJobs.createOrReuse, jobArgs({ + roomId, + artifactId, + proof, + idempotencyKey: "job-runtime-workflow-concurrent-not-claimed", + })); + await t.run((ctx) => ctx.db.patch(concurrent.jobId, { + status: "running" as const, + attempts: 2, + workflowId: "workflow-concurrent", + leaseId: "lease-concurrent", + leaseUntil: Date.now() + 60_000, + })); + + expect(await t.mutation(internal.agentJobs.recordWorkflowComplete, { + jobId: concurrent.jobId, + workflowId: "workflow-concurrent", + resultKind: "failed" as const, + error: "agent_slice_failed:not_claimed", + })).toEqual({ ok: true, terminal: false, superseded: true }); + const concurrentDetail = await t.query(api.agentJobs.detail, { jobId: concurrent.jobId, requester: proof }); + expect(concurrentDetail?.job).toMatchObject({ status: "running", leaseId: "lease-concurrent" }); + }); + + it("replays an exact committed finishSlice response without duplicating the attempt or accounting", async () => { + const { t, proof, roomId, artifactId } = await setupRoom(); + const { jobId } = await t.mutation(api.agentJobs.createOrReuse, jobArgs({ + roomId, + artifactId, + proof, + idempotencyKey: "job-runtime-finish-response-replay", + })); + await t.mutation(internal.agentJobs.claimSlice, { jobId, leaseId: "lease-finish-replay", leaseMs: 60_000 }); + const runId = await t.mutation(internal.agentRuns.record, { + jobId, + roomId, + agentId: "agent_pub", + model: "test-model", + goal: "finish exactly once", + steps: 1, + modelCalls: 1, + toolCalls: 1, + conflictsSurvived: 0, + inputTokens: 10, + outputTokens: 5, + costUsd: 0.0001, + costKind: "exact" as const, + ms: 100, + exhausted: false, + stopReason: "done", + }); + const args = { ...finishSliceArgs({ jobId, leaseId: "lease-finish-replay", attempt: 1 }), runId }; + + expect(await t.mutation(internal.agentJobs.finishSlice, args)).toEqual({ ok: true }); + expect(await t.mutation(internal.agentJobs.finishSlice, args)).toEqual({ ok: true, reused: true }); + const detail = await t.query(api.agentJobs.detail, { jobId, requester: proof }); + expect(detail?.attempts).toHaveLength(1); + expect(detail?.job).toMatchObject({ modelCallCount: 1, toolCallCount: 1, inputTokens: 10, outputTokens: 5 }); + expect(detail?.job.costUsd).toBeCloseTo(0.0001); + }); + it("finishSlice preserves failover call counts and cache cost provenance", async () => { const { t, proof, roomId, artifactId } = await setupRoom(); const { jobId } = await t.mutation(api.agentJobs.createOrReuse, jobArgs({ roomId, artifactId, proof, idempotencyKey: "job-runtime-exact-accounting" })); @@ -1481,6 +1670,62 @@ describe("agentJobs runtime contract", () => { expect(completed?.job.costUsd).toBeCloseTo(0.005); }); + it("finishSlice carries forward cumulative run accounting from a pre-checkpoint crash", async () => { + const { t, proof, roomId, artifactId } = await setupRoom(); + const { jobId } = await t.mutation(api.agentJobs.createOrReuse, jobArgs({ + roomId, + artifactId, + proof, + idempotencyKey: "job-runtime-orphan-run-accounting", + })); + await t.mutation(internal.agentJobs.claimSlice, { jobId, leaseId: "lease-orphan-accounting", leaseMs: 60_000 }); + await t.run((ctx) => ctx.db.patch(jobId, { + modelCallCount: 3, + toolCallCount: 5, + inputTokens: 130, + outputTokens: 50, + cachedInputTokens: 25, + cacheCreationInputTokens: 4, + costUsd: 0.012, + costKind: "estimated" as const, + })); + + await t.mutation(internal.agentJobs.finishSlice, { + ...finishSliceArgs({ jobId, leaseId: "lease-orphan-accounting", attempt: 1 }), + modelCalls: 1, + toolCalls: 2, + inputTokens: 30, + outputTokens: 10, + cachedInputTokens: 5, + cacheCreationInputTokens: 0, + costUsd: 0.002, + costKind: "exact" as const, + accountingTotals: { + modelCalls: 3, + toolCalls: 5, + inputTokens: 130, + outputTokens: 50, + cachedInputTokens: 25, + cacheCreationInputTokens: 4, + costUsd: 0.012, + costKind: "estimated" as const, + }, + }); + + const detail = await t.query(api.agentJobs.detail, { jobId, requester: proof }); + expect(detail?.job).toMatchObject({ + modelCallCount: 3, + toolCallCount: 5, + inputTokens: 130, + outputTokens: 50, + cachedInputTokens: 25, + cacheCreationInputTokens: 4, + costKind: "estimated", + }); + expect(detail?.job.costUsd).toBeCloseTo(0.012); + expect(detail?.attempts[0]).toMatchObject({ modelCalls: 1, toolCalls: 2, inputTokens: 30, costUsd: 0.002 }); + }); + it("finishSlice reconstructs a legacy attempt baseline before adding the current slice", async () => { const { t, proof, roomId, artifactId } = await setupRoom(); const { jobId } = await t.mutation(api.agentJobs.createOrReuse, jobArgs({ roomId, artifactId, proof, idempotencyKey: "job-runtime-slice-aggregate-migration" })); @@ -1739,6 +1984,38 @@ describe("agentJobs runtime contract", () => { expect(detail?.receipts).toHaveLength(1); }); + it("does not let an interactive finisher overwrite a cancelled job", async () => { + const { t, proof, roomId, artifactId } = await setupRoom(); + const { jobId } = await t.mutation(api.agentJobs.createOrReuse, jobArgs({ + roomId, + artifactId, + proof, + idempotencyKey: "job-runtime-interactive-cancel-fence", + })); + await t.mutation(api.agentJobs.cancel, { jobId, requester: proof }); + + const finish = await t.mutation(internal.agentJobs.finishInteractive, { + jobId, + status: "completed", + finalText: "stale interactive completion", + resolvedModel: "test-model", + stopReason: "done", + ms: 100, + inputTokens: 10, + outputTokens: 5, + costUsd: 0.0001, + costKind: "exact" as const, + modelCalls: 1, + toolCalls: 0, + }); + const detail = await t.query(api.agentJobs.detail, { jobId, requester: proof }); + + expect(finish).toEqual({ ok: true, terminal: true }); + expect(detail?.job).toMatchObject({ status: "cancelled", error: "cancelled_by_user" }); + expect(detail?.job.finalText).not.toContain("stale interactive completion"); + expect(detail?.attempts).toHaveLength(0); + }); + it("retry requeues a cancelled job with a fresh workflow and fences the old lease", async () => { vi.useFakeTimers(); try { diff --git a/tests/agentJobsSource.test.ts b/tests/agentJobsSource.test.ts index 19856036..f89a846e 100644 --- a/tests/agentJobsSource.test.ts +++ b/tests/agentJobsSource.test.ts @@ -52,11 +52,16 @@ describe("long-running agent job source invariants", () => { expect(workflows).toContain("new WorkflowManager(components.workflow"); expect(workflows).toContain("MAX_WORKFLOW_SLICES"); expect(workflows).toContain("One workflow invocation owns one long-running slice"); + expect(workflows.match(/requireSuccessfulSliceResult\(await step\.runAction/g)).toHaveLength(2); + expect(workflows).toContain("agent_slice_failed:"); expect(jobs).toContain("agentWorkflows.freeAutoWorkflow.continue"); expect(jobs).toContain('job.status === "paused" || job.status === "retrying"'); expect(jobs).toContain('resultKind === "success" && job.status === "queued"'); expect(jobs).toContain("agentJobs.finishSlice.workflowSchedulerFallback"); - expect(jobs).toContain('job.status === "running" && job.attempts > 1'); + expect(jobs).toContain('error?.includes("agent_slice_failed:not_claimed")'); + expect(jobs).toContain('job.status === "running"'); + expect(jobs).toContain('job.leaseUntil > Date.now()'); + expect(jobs).not.toContain('job.status === "running" && job.attempts > 1'); }); it("expands spreadsheet locks through formula dependency records", () => { @@ -329,16 +334,37 @@ describe("long-running agent job source invariants", () => { const journalClient = readFileSync("convex/agentStepJournalClient.ts", "utf8"); const agent = readFileSync("convex/agent.ts", "utf8"); const runner = readFileSync("convex/agentJobRunner.ts", "utf8"); + const runs = readFileSync("convex/agentRuns.ts", "utf8"); + const stepChain = readFileSync("convex/agentStepChain.ts", "utf8"); const journal = readFileSync("src/nodeagent/core/journal.ts", "utf8"); expect(schema).toContain("agentModelStepJournal"); expect(schema).toContain('index("by_job_slice_step", ["jobId", "sliceKey", "step"])'); - expect(journalFns).toContain("export const get = internalQuery"); + expect(journalFns).toContain("export const get = internalMutation"); expect(journalFns).toContain("export const record = internalMutation"); + expect(journalFns).toContain("requireJournalLease"); + expect(journalFns).toContain("job_lease_invalid"); + expect(journalFns).toContain("journal_replay_mismatch"); expect(journalClient).toContain("makeConvexStepJournal"); + expect(journalClient).toContain('makeFunctionReference<"mutation">("agentStepJournal:get")'); + expect(journalClient).toContain("leaseId?: string"); expect(journal).toContain("journalSliceKey"); expect(agent).toContain("journal: modelJournal"); expect(runner).toContain("journal: modelJournal"); + expect(runner).toMatch(/makeConvexStepJournal\(\{[\s\S]*?jobId: claimed\.jobId,[\s\S]*?leaseId,/); + expect(journalClient).toContain("accountingClaims()"); + expect(runner).toContain("agentRuns:recordJournaled"); + expect(runner).toContain("journalClaims = modelJournal.accountingClaims()"); + expect(journalClient).toContain('state: "confirmed" | "pending"'); + expect(runner).toContain("traceSteps,"); + expect(runner).toContain("const canRetry = !runAlreadyPersisted"); + expect(runs).toContain('leaseId: v.string()'); + expect(runs).toContain('if (a.traceSteps.length === 0) throw new Error("agent_run_trace_empty")'); + expect(runs).toContain("recordAgentStepChain(ctx"); + expect(stepChain).toContain("trace_count_unavailable"); + expect(runner).toContain("accountingTotals: jobAccounting"); + expect(schema).toContain("accountedRunId: v.optional(v.id(\"agentRuns\"))"); + expect(schema).toContain("traceRecordCount: v.optional(v.number())"); }); it("defines the operation ledger, receipts, draft operations, and first-class leases", () => { @@ -347,6 +373,7 @@ describe("long-running agent job source invariants", () => { const artifacts = readFileSync("convex/artifacts.ts", "utf8"); const roomTools = readFileSync("convex/convexRoomTools.ts", "utf8"); const steps = readFileSync("convex/agentSteps.ts", "utf8"); + const stepChain = readFileSync("convex/agentStepChain.ts", "utf8"); expect(schema).toContain("agentOperationEvents"); expect(schema).toContain("agentStreamEvents"); @@ -362,7 +389,7 @@ describe("long-running agent job source invariants", () => { expect(artifacts).toContain('jobId: v.optional(v.id("agentJobs"))'); expect(roomTools).toContain("private jobId?: Id<\"agentJobs\">"); expect(roomTools).toContain("jobId: this.jobId"); - expect(steps).toContain("mutationReceiptIds"); + expect(stepChain).toContain("mutationReceiptIds"); expect(steps).toContain('jobId: v.optional(v.id("agentJobs"))'); }); diff --git a/tests/convexModelFreeAutoMode.test.ts b/tests/convexModelFreeAutoMode.test.ts index ba875a94..83222dd1 100644 --- a/tests/convexModelFreeAutoMode.test.ts +++ b/tests/convexModelFreeAutoMode.test.ts @@ -178,7 +178,6 @@ describe("Convex free-auto model routing", () => { attempted.push(String(body.model)); return new Response(JSON.stringify({ choices: [{ message: { content: "I will make that change." } }], - usage: { prompt_tokens: 10, completion_tokens: 5 }, }), { status: 200, headers: { "Content-Type": "application/json" } }); })); @@ -200,6 +199,13 @@ describe("Convex free-auto model routing", () => { expect(route.name).toBe("nvidia/nemotron-3-super-120b-a12b:free"); expect(step.text).toBe("I will make that change."); expect(step.toolCalls).toEqual([]); + expect(step.usage).toMatchObject({ + inputTokens: 0, + outputTokens: 0, + modelCalls: 1, + costUsd: 0, + costKind: "exact", + }); expect(step.providerRoute).toMatchObject({ requestedModel: "nvidia/nemotron-3-super-120b-a12b:free", resolvedModel: "nvidia/nemotron-3-super-120b-a12b:free", @@ -457,7 +463,7 @@ describe("Convex free-auto model routing", () => { expect(result.text).toBe("fallback answer"); }); - it("does not label unknown catalog pricing as exact cost", async () => { + it("retains a conservative reservation when catalog pricing cannot be measured exactly", async () => { vi.stubGlobal("fetch", vi.fn(async () => new Response(JSON.stringify({ choices: [{ message: { content: "answer" } }], usage: { prompt_tokens: 5, completion_tokens: 2 }, @@ -467,8 +473,8 @@ describe("Convex free-auto model routing", () => { messages: [{ role: "user", content: "hello" }], tools: [], }); - expect(step.usage).toMatchObject({ inputTokens: 5, outputTokens: 2, modelCalls: 1 }); - expect(step.usage).not.toHaveProperty("costUsd"); + expect(step.usage).toMatchObject({ inputTokens: 5, outputTokens: 2, modelCalls: 1, costKind: "estimated" }); + expect(step.usage?.costUsd).toBeGreaterThan(0); const result = await runAgent({ rt: runtimeTools(), @@ -482,6 +488,82 @@ describe("Convex free-auto model routing", () => { expect(result.usage.costUsd).toBeGreaterThan(0); }); + it("blocks an unknown paid route before the provider call when a hard turn budget is present", async () => { + const fetchSpy = vi.fn(); + vi.stubGlobal("fetch", fetchSpy); + + const error = await convexModel("vendor/unknown-paid-model", { entrypoint: "public_ask" }).next({ + system: "Answer.", + messages: [{ role: "user", content: "hello" }], + tools: [], + maxCostUsd: 0.01, + }).then(() => undefined, (failure: unknown) => failure); + + expect(error).toBeInstanceOf(QualityFailoverError); + expect(fetchSpy).not.toHaveBeenCalled(); + expect((error as QualityFailoverError).receipt).toMatchObject({ + stopReason: "spend_budget", + routeAttempts: [], + skippedRoutes: [{ routeId: "vendor/unknown-paid-model", reason: "spend_budget" }], + }); + }); + + it("keeps cumulative paid failover reservations inside the remaining turn budget", async () => { + process.env.NEBIUS_API_KEY = "test-nebius-key"; + const attempted: string[] = []; + vi.stubGlobal("fetch", vi.fn(async (_input: string | URL | Request, init?: RequestInit) => { + const body = JSON.parse(String(init?.body ?? "{}")) as { model?: string }; + attempted.push(String(body.model)); + return new Response("primary temporarily unavailable", { status: 503 }); + })); + + const error = await convexModel("nebius/zai-org/GLM-5.2", { + entrypoint: "public_ask", + fallbackModelIds: ["z-ai/glm-5.2"], + }).next({ + system: "Answer.", + messages: [{ role: "user", content: "hello" }], + tools: [], + maxCostUsd: 0.03, + }).then(() => undefined, (failure: unknown) => failure); + + expect(error).toBeInstanceOf(QualityFailoverError); + expect(attempted).toEqual(["zai-org/GLM-5.2"]); + expect((error as QualityFailoverError).receipt).toMatchObject({ + stopReason: "spend_budget", + routeAttempts: [{ routeId: "nebius/zai-org/GLM-5.2", outcome: "provider_failure" }], + skippedRoutes: [{ routeId: "z-ai/glm-5.2", reason: "spend_budget" }], + }); + expect((error as QualityFailoverError).usage).toMatchObject({ modelCalls: 1, costKind: "estimated" }); + }); + + it("blocks a paid route before calling it when its reservation exceeds the remaining run budget", async () => { + process.env.NEBIUS_API_KEY = "test-nebius-key"; + const fetchSpy = vi.fn(async () => new Response(JSON.stringify({ + choices: [{ message: { content: "I will update it." } }], + }), { status: 200, headers: { "Content-Type": "application/json" } })); + vi.stubGlobal("fetch", fetchSpy); + + const result = await runAgent({ + rt: runtimeTools(), + goal: "write 42 into the workbook cells", + model: convexModel("nebius/zai-org/GLM-5.2", { entrypoint: "public_ask" }), + tools: ROOM_TOOLS, + maxSteps: 8, + spendLimits: { maxCostUsd: 0.001 }, + }); + + expect(fetchSpy).not.toHaveBeenCalled(); + expect(result.stopReason).toBe("spend_budget"); + expect(result.usage).toMatchObject({ + inputTokens: 0, + outputTokens: 0, + modelCalls: 0, + costKind: "exact", + }); + expect(result.usage.costUsd).toBe(0); + }); + it("counts no provider call when a direct provider key is missing", async () => { const fetchSpy = vi.fn(); vi.stubGlobal("fetch", fetchSpy); @@ -524,13 +606,17 @@ describe("Convex free-auto model routing", () => { const error = await pending; expect(error).toBeInstanceOf(QualityFailoverError); - expect((error as QualityFailoverError).usage).toMatchObject({ + const timeoutUsage = (error as QualityFailoverError).usage; + expect(timeoutUsage).toMatchObject({ inputTokens: 0, outputTokens: 0, modelCalls: 1, - costUsd: 0, costKind: "estimated", }); + expect(timeoutUsage?.costUsd).toBeGreaterThan(0); + const timeoutAttempt = (error as QualityFailoverError).receipt.routeAttempts[0]; + expect(timeoutAttempt?.estimatedCostUsd).toBeGreaterThan(0); + expect(timeoutAttempt?.costUsd).toBe(timeoutUsage?.costUsd); expect(route.routeState?.()).toMatchObject({ cooldownUntil: { "nebius/zai-org/GLM-5.2": expect.any(Number) }, }); diff --git a/tests/gatewayAndJournal.test.ts b/tests/gatewayAndJournal.test.ts index 0c478bfb..1a7dfd7b 100644 --- a/tests/gatewayAndJournal.test.ts +++ b/tests/gatewayAndJournal.test.ts @@ -4,7 +4,7 @@ import { describe, it, expect } from "vitest"; import { checkSpendCeiling, redactPII } from "../src/nodeagent/guardrails/gateway"; import { journalSliceKey, MapStepJournal } from "../src/nodeagent/core/journal"; -import { runAgent } from "../src/nodeagent/core/runtime"; +import { AgentRunError, runAgent } from "../src/nodeagent/core/runtime"; import { scriptedModel } from "../src/nodeagent/models/scripted"; import { recomputeVariancePlan } from "../src/nodeagent/core/plans"; import { RoomEngine } from "../src/engine/roomEngine"; @@ -12,6 +12,7 @@ import { buildDemoRoom } from "../src/engine/demoRoom"; import { InMemoryRoomTools } from "../src/nodeagent/skills/integration/noderoomAdapter"; import { ROOM_TOOLS } from "../src/nodeagent/skills/spreadsheet/cellMutator"; import type { AgentHandoff, AgentModel } from "../src/nodeagent/core/types"; +import { makeConvexStepJournal } from "../convex/agentStepJournalClient"; const CELL = "r_ni__variance"; const VAL = "+22.4%"; @@ -117,6 +118,71 @@ describe("P1-3: error-path handoff preserves unexecuted tool calls (resume-curso }); describe("exactly-once journal (no double-bill on slice retry)", () => { + it("derives the same per-step accounting claim for a committed step and its crash replay", async () => { + const stored = new Map>>(); + const calls: Array> = []; + const ctx = { + runMutation: async (_ref: unknown, args: Record) => { + if (!("result" in args)) return stored.get(Number(args.step)); + calls.push(args); + stored.set(Number(args.step), args.result as Awaited>); + }, + }; + const result = { + text: "done", + toolCalls: [], + done: true, + usage: { inputTokens: 10, outputTokens: 4, modelCalls: 2, costUsd: 0.01, costKind: "exact" as const }, + }; + const fresh = makeConvexStepJournal({ + ctx, + jobId: "job1" as never, + leaseId: "lease-1", + sliceKey: "slice-1", + modelName: () => "provider/model", + }); + await fresh.record(0, result); + const freshClaims = fresh.accountingClaims(); + + const replay = makeConvexStepJournal({ + ctx, + jobId: "job1" as never, + leaseId: "lease-2", + sliceKey: "slice-1", + modelName: () => "provider/model", + }); + expect(await replay.get(0)).toEqual(result); + + expect(replay.accountingClaims()).toEqual(freshClaims); + expect(freshClaims).toEqual([{ step: 0, outputHash: expect.any(String), state: "confirmed" }]); + expect(calls[0]).toMatchObject({ leaseId: "lease-1", outputHash: expect.any(String) }); + }); + + it("retains a pending claim when a journal mutation may have committed before throwing", async () => { + const result = { + text: "done", + toolCalls: [], + done: true, + usage: { inputTokens: 10, outputTokens: 4, modelCalls: 1, costUsd: 0.01, costKind: "exact" as const }, + }; + const journal = makeConvexStepJournal({ + ctx: { + runMutation: async () => { throw new Error("response_lost_after_commit"); }, + }, + jobId: "job1" as never, + leaseId: "lease-1", + sliceKey: "slice-1", + modelName: () => "provider/model", + }); + + await expect(journal.record(0, result)).rejects.toThrow("response_lost_after_commit"); + expect(journal.accountingClaims()).toEqual([{ + step: 0, + outputHash: expect.any(String), + state: "pending", + }]); + }); + it("derives stable slice keys from semantic input, not object insertion order", () => { const a = journalSliceKey({ jobId: "job1", cursor: { b: 2, a: 1 }, stepBudget: 3 }); const b = journalSliceKey({ stepBudget: 3, cursor: { a: 1, b: 2 }, jobId: "job1" }); @@ -189,8 +255,17 @@ describe("exactly-once journal (no double-bill on slice retry)", () => { }, }; - await expect(runAgent({ rt, goal: "Summarize the workbook.", model, tools: [], maxSteps: 2, journal })) - .rejects.toThrow("simulated_crash_after_journal_commit"); + const failed = await runAgent({ rt, goal: "Summarize the workbook.", model, tools: [], maxSteps: 2, journal }) + .then(() => undefined, (error: unknown) => error); + expect(failed).toBeInstanceOf(AgentRunError); + expect((failed as AgentRunError).message).toContain("simulated_crash_after_journal_commit"); + expect((failed as AgentRunError).partial.usage).toMatchObject({ + inputTokens: 17, + outputTokens: 5, + modelCalls: 1, + costUsd: 0.004, + costKind: "exact", + }); expect(providerRequests).toBe(1); crashAfterCommit = false; diff --git a/tests/idempotencyRuntime.test.ts b/tests/idempotencyRuntime.test.ts index 6f561580..44c74f80 100644 --- a/tests/idempotencyRuntime.test.ts +++ b/tests/idempotencyRuntime.test.ts @@ -9,6 +9,7 @@ import { expect, test } from "vitest"; import schema from "../convex/schema"; import { internal } from "../convex/_generated/api"; import { runIdempotencyKey, findReusableRun } from "../src/nodeagent/core/idempotency"; +import { verifyAgentStepChain } from "../convex/agentStepChain"; // In-memory modules, EXCLUDING the "use node" action (AI SDK; not needed for the dedup data layer, won't load in edge-runtime). const modules = import.meta.glob("../convex/**/*.ts"); @@ -118,3 +119,243 @@ test("RUNTIME: agentRuns.record preserves explicit zero calls and cache accounti costKind: "exact", }); }); + +test("RUNTIME: overlapping journal retries claim each provider step once and preserve unjournaled residual spend", async () => { + const t = convexTest(schema, modules); + const { roomId, jobId } = await t.run(async (ctx) => { + const now = Date.now(); + const roomId = await ctx.db.insert("rooms", { code: "TEST04", title: "Replay accounting", hostId: "u1", autoAllow: true, status: "live" as const, createdAt: now }); + const artifactId = await ctx.db.insert("artifacts", { roomId, kind: "sheet" as const, title: "Replay", version: 1, order: [], updatedAt: now }); + const jobId = await ctx.db.insert("agentJobs", { + roomId, + artifactId, + requester: { kind: "user" as const, id: "u1", name: "User" }, + goal: "Finish a journal-backed workbook slice", + status: "running" as const, + modelPolicy: "provider/model", + attempts: 1, + maxAttempts: 4, + leaseId: "lease-record", + leaseUntil: now + 60_000, + createdAt: now, + updatedAt: now, + }); + const entries = [ + { step: 0, outputHash: "hash-a", usage: { inputTokens: 100, outputTokens: 40, cachedInputTokens: 20, cacheCreationInputTokens: 0, modelCalls: 1, costUsd: 0.01, costKind: "exact" as const } }, + { step: 1, outputHash: "hash-b", usage: { inputTokens: 120, outputTokens: 50, cachedInputTokens: 0, cacheCreationInputTokens: 0, modelCalls: 1, costUsd: 0.02, costKind: "exact" as const } }, + ]; + for (const entry of entries) { + await ctx.db.insert("agentModelStepJournal", { + jobId, + sliceKey: "slice-1", + step: entry.step, + model: "provider/model", + inputHash: "slice-1", + outputHash: entry.outputHash, + result: { text: `step-${entry.step}`, toolCalls: [], done: false, usage: entry.usage }, + createdAt: now, + updatedAt: now, + }); + } + return { roomId, jobId }; + }); + const base = { + jobId, + roomId, + leaseId: "lease-record", + attempt: 1, + journalSliceKey: "slice-1", + agentId: "agent_pub", + model: "provider/model", + goal: "Finish a journal-backed workbook slice", + steps: 3, + traceSteps: [{ + idx: 0, + tool: "model_result", + args: "{}", + result: "done", + status: "ok" as const, + ms: 50, + }], + toolCalls: 4, + conflictsSurvived: 0, + costKind: "exact" as const, + ms: 900, + exhausted: false, + stopReason: "done", + }; + + const first = await t.mutation(internal.agentRuns.recordJournaled, { + ...base, + recordKey: "durable-run:first", + traceSteps: [{ + idx: 0, + tool: "inspect_workbook", + args: "{}", + result: "inspected", + status: "ok" as const, + ms: 50, + affectedObjectIds: ["cell:A1"], + }], + // Pending + matching durable row models journal commit followed by a lost response. + journalClaims: [{ step: 0, outputHash: "hash-a", state: "pending" as const }], + modelCalls: 1, + inputTokens: 100, + outputTokens: 40, + cachedInputTokens: 20, + cacheCreationInputTokens: 0, + costUsd: 0.01, + }); + const secondArgs = { + ...base, + recordKey: "durable-run:second", + journalClaims: [ + { step: 0, outputHash: "hash-a", state: "confirmed" as const }, + { step: 1, outputHash: "hash-b", state: "confirmed" as const }, + ], + // Logical usage rematerializes A+B, plus one unjournaled failed provider request. + modelCalls: 3, + inputTokens: 230, + outputTokens: 95, + cachedInputTokens: 20, + cacheCreationInputTokens: 0, + costUsd: 0.035, + }; + const second = await t.mutation(internal.agentRuns.recordJournaled, secondArgs); + const exactReplay = await t.mutation(internal.agentRuns.recordJournaled, secondArgs); + const roomSpend = await t.query(internal.agentRuns.roomSpendSince, { roomId, since: 0 }); + const globalSpend = await t.query(internal.agentRuns.globalSpendSince, { since: 0 }); + const journalRows = await t.run((ctx) => ctx.db.query("agentModelStepJournal").withIndex("by_job", (q) => q.eq("jobId", jobId)).collect()); + const job = await t.run((ctx) => ctx.db.get(jobId)); + const firstRun = await t.run((ctx) => ctx.db.get(first.runId)); + const firstTraceRows = await t.run((ctx) => ctx.db.query("agentSteps").withIndex("by_run", (q) => q.eq("runId", first.runId)).collect()); + + expect(first.accounting).toMatchObject({ modelCalls: 1, inputTokens: 100, outputTokens: 40, costUsd: 0.01 }); + // A is replayed and excluded. The second row contains B plus only the real unjournaled failure. + expect(second.accounting).toMatchObject({ modelCalls: 2, inputTokens: 130, outputTokens: 55 }); + expect(second.accounting.costUsd).toBeCloseTo(0.025); + expect(String(exactReplay.runId)).toBe(String(second.runId)); + expect(exactReplay.reused).toBe(true); + expect(second.jobAccounting).toMatchObject({ modelCalls: 3, inputTokens: 230, outputTokens: 95 }); + expect(second.jobAccounting.costUsd).toBeCloseTo(0.035); + expect(job).toMatchObject({ modelCallCount: 3, toolCallCount: 8, inputTokens: 230, outputTokens: 95, costKind: "exact" }); + expect(job?.costUsd).toBeCloseTo(0.035); + expect(roomSpend).toBeCloseTo(0.035); + expect(globalSpend.totalUsd).toBeCloseTo(0.035); + expect(globalSpend.runCount).toBe(2); + expect(journalRows.every((row) => row.accountedRunId)).toBe(true); + expect(firstRun?.traceRecordCount).toBe(1); + expect(firstTraceRows).toHaveLength(1); + expect(await verifyAgentStepChain({ + jobId, + runId: first.runId, + roomId, + agentId: "agent_pub", + expectedCount: firstRun?.traceRecordCount, + rows: firstTraceRows, + hashes: firstTraceRows, + })).toEqual({ valid: true, steps: 1 }); + const tamperedTrace = firstTraceRows.map((row) => ({ ...row, affectedObjectIds: ["cell:B9"] })); + expect(await verifyAgentStepChain({ + jobId, + runId: first.runId, + roomId, + agentId: "agent_pub", + expectedCount: firstRun?.traceRecordCount, + rows: tamperedTrace, + hashes: firstTraceRows, + })).toMatchObject({ valid: false, reason: "record hash mismatch - tampered" }); + const wrongJobTrace = firstTraceRows.map((row) => ({ ...row, jobId: undefined })); + expect(await verifyAgentStepChain({ + jobId, + runId: first.runId, + roomId, + agentId: "agent_pub", + expectedCount: firstRun?.traceRecordCount, + rows: wrongJobTrace, + hashes: firstTraceRows, + })).toMatchObject({ valid: false, reason: "trace_identity_mismatch" }); + + await expect(t.mutation(internal.agentRuns.recordJournaled, { + ...secondArgs, + recordKey: "durable-run:mismatch", + journalClaims: [{ step: 0, outputHash: "wrong", state: "confirmed" as const }], + })).rejects.toThrow("agent_run_journal_claim_mismatch"); + + await t.run((ctx) => ctx.db.patch(jobId, { + leaseId: "lease-record-2", + leaseUntil: Date.now() + 60_000, + })); + await expect(t.mutation(internal.agentRuns.recordJournaled, { + ...secondArgs, + recordKey: "durable-run:stale-lease", + })).rejects.toThrow("agent_run_lease_invalid"); + + await t.run((ctx) => ctx.db.patch(jobId, { attempts: 2 })); + await expect(t.mutation(internal.agentRuns.recordJournaled, { + ...secondArgs, + leaseId: "lease-record-2", + recordKey: "durable-run:stale-attempt", + })).rejects.toThrow("agent_run_attempt_mismatch"); + await expect(t.mutation(internal.agentRuns.recordJournaled, { + ...secondArgs, + leaseId: "lease-record-2", + attempt: 2, + recordKey: "durable-run:empty-trace", + traceSteps: [], + })).rejects.toThrow("agent_run_trace_empty"); +}); + +test("RUNTIME: agent step trace replay is idempotent and rejects divergent chains", async () => { + const t = convexTest(schema, modules); + const roomId = await t.run((ctx) => + ctx.db.insert("rooms", { code: "TEST05", title: "Trace replay", hostId: "u1", autoAllow: true, status: "live" as const, createdAt: Date.now() })); + const runId = await t.mutation(internal.agentRuns.record, { + roomId, + agentId: "agent_pub", + model: "provider/model", + goal: "Trace once", + steps: 1, + modelCalls: 1, + toolCalls: 1, + conflictsSurvived: 0, + inputTokens: 10, + outputTokens: 5, + costUsd: 0.001, + costKind: "estimated", + ms: 100, + exhausted: false, + stopReason: "done", + }); + const args = { + runId, + roomId, + agentId: "agent_pub", + steps: [{ idx: 0, tool: "inspect_workbook", args: "{}", result: "ok", status: "ok" as const, ms: 50 }], + }; + const incompleteRun = await t.run((ctx) => ctx.db.get(runId)); + expect(await verifyAgentStepChain({ + runId, + roomId, + agentId: "agent_pub", + expectedCount: incompleteRun?.traceRecordCount, + rows: [], + hashes: [], + })).toMatchObject({ valid: false, reason: "trace_count_unavailable" }); + expect(await verifyAgentStepChain({ + runId, + roomId, + agentId: "agent_pub", + expectedCount: 0, + rows: [], + hashes: [], + })).toMatchObject({ valid: false, reason: "trace_empty" }); + expect(await t.mutation(internal.agentSteps.record, args)).toEqual({ reused: false, inserted: 1 }); + expect(await t.mutation(internal.agentSteps.record, args)).toEqual({ reused: true, inserted: 0 }); + const rows = await t.run((ctx) => ctx.db.query("agentSteps").withIndex("by_run", (q) => q.eq("runId", runId)).collect()); + expect(rows).toHaveLength(1); + await expect(t.mutation(internal.agentSteps.record, { + ...args, + steps: [{ ...args.steps[0], result: "different" }], + })).rejects.toThrow("agent_steps_replay_mismatch"); +}); diff --git a/tests/qualityFailover.test.ts b/tests/qualityFailover.test.ts index 7519c97d..29e27e1b 100644 --- a/tests/qualityFailover.test.ts +++ b/tests/qualityFailover.test.ts @@ -285,6 +285,27 @@ describe("quality-aware bounded failover", () => { }); }); + it("retains the pre-call reservation when measured provider cost is unavailable", async () => { + const result = await runQualityFailover({ + candidates: [candidate("usage-missing", "accepted", { estimatedCostUsd: 0.25 })], + budget: { maxAttempts: 1, maxCostUsd: 0.5 }, + execute: async (route) => route.response, + measureCostUsd: () => undefined, + now: () => 4_500, + }); + + expect(result.ok).toBe(true); + expect(result.receipt).toMatchObject({ + routeAttempts: [{ + routeId: "usage-missing", + estimatedCostUsd: 0.25, + costUsd: 0.25, + outcome: "accepted", + }], + budget: { spentCostUsd: 0.25, remainingCostUsd: 0.25 }, + }); + }); + it("enforces the attempt ceiling before calling another candidate", async () => { const execute = vi.fn(async (route: Candidate) => route.response); const result = await runQualityFailover({ diff --git a/tests/roomShellTelemetry.test.ts b/tests/roomShellTelemetry.test.ts index c22db63f..1a1d9d14 100644 --- a/tests/roomShellTelemetry.test.ts +++ b/tests/roomShellTelemetry.test.ts @@ -1,6 +1,11 @@ import { describe, expect, it } from "vitest"; -import { projectAgentJobAttemptTelemetry, projectAgentRunTelemetry } from "../src/app/store"; -import { formatAgentCost } from "../src/ui/RoomShell"; +import { + projectAgentJobAttemptTelemetry, + projectAgentRunTelemetry, + selectAgentRunTelemetryForJob, + type AgentJobTelemetry, +} from "../src/app/store"; +import { formatAgentCost, selectedJobSignalTelemetry } from "../src/ui/RoomShell"; const persistedRun = { model: "provider/known-model", @@ -12,6 +17,18 @@ const persistedRun = { ms: 400, }; +function durableJob(overrides: Partial = {}): AgentJobTelemetry { + return { + id: "job-selected", + status: "running", + attempts: 1, + maxAttempts: 3, + modelPolicy: "openrouter/free-auto", + updatedAt: 100, + ...overrides, + }; +} + describe("RoomShell cost telemetry", () => { it("preserves estimated run pricing through the store projection and labels it approximately", () => { const run = projectAgentRunTelemetry({ ...persistedRun, costKind: "estimated" }); @@ -45,3 +62,95 @@ describe("RoomShell cost telemetry", () => { expect(formatAgentCost(run.costUsd, run.costKind)).toBe("≈$0.125"); }); }); + +describe("RoomShell selected-job telemetry", () => { + it("displays the selected job's run when a different concurrent job owns the room's newest run", () => { + const selectedRun = { + ...persistedRun, + _id: "run-selected", + jobId: "job-selected", + model: "provider/selected-model", + toolCalls: 3, + costUsd: 0.031, + costKind: "exact" as const, + }; + const otherRun = { + ...persistedRun, + _id: "run-other", + jobId: "job-other", + model: "provider/other-model", + toolCalls: 9, + costUsd: 0.909, + costKind: "exact" as const, + }; + const job = durableJob({ latestRunId: "run-selected" }); + + const run = selectAgentRunTelemetryForJob([otherRun, selectedRun], job); + + expect(run).toMatchObject({ model: "provider/selected-model", toolCalls: 3, costUsd: 0.031 }); + expect(selectedJobSignalTelemetry(job, run)).toEqual({ + evalValue: "provider/selected-model | 3 tools", + costValue: "$0.031", + }); + }); + + it("uses selected-job detail when its latest run has fallen outside the bounded room run list", () => { + const detailRun = { + ...persistedRun, + _id: "run-selected", + jobId: "job-selected", + model: "provider/detail-model", + costKind: "estimated" as const, + }; + const job = durableJob({ latestRunId: "run-selected" }); + + const run = selectAgentRunTelemetryForJob([ + { ...persistedRun, _id: "run-other", jobId: "job-other", model: "provider/other-model" }, + ], job, { + job: { _id: "job-selected", latestRunId: "run-selected" }, + latestRun: detailRun, + }); + + expect(run).toMatchObject({ model: "provider/detail-model", costKind: "estimated" }); + }); + + it("accepts a legacy run without jobId only when latestRunId provides the durable link", () => { + const job = durableJob({ latestRunId: "run-legacy" }); + + const run = selectAgentRunTelemetryForJob([ + { ...persistedRun, _id: "run-legacy", model: "provider/legacy-model" }, + ], job); + + expect(run?.model).toBe("provider/legacy-model"); + }); + + it("withholds uncorrelated legacy telemetry and reports only selected-job aggregate data", () => { + const job = durableJob({ + latestRunId: undefined, + modelPolicy: "openrouter/free-auto", + toolCallCount: 4, + costUsd: 0.2, + }); + const unrelatedLegacyRun = { ...persistedRun, _id: "run-unknown", model: "provider/unrelated-model" }; + + const run = selectAgentRunTelemetryForJob([unrelatedLegacyRun], job, { + job: { _id: "job-other", latestRunId: "run-unknown" }, + latestRun: unrelatedLegacyRun, + }); + + expect(run).toBeNull(); + expect(selectedJobSignalTelemetry(job, run)).toEqual({ + evalValue: "route openrouter/free-auto | 4 tools total", + costValue: "≈$0.200 job total", + }); + }); + + it("labels selected-job telemetry as not reported when legacy counters are missing", () => { + const job = durableJob({ modelPolicy: "legacy/policy", latestRunId: undefined }); + + expect(selectedJobSignalTelemetry(job, null)).toEqual({ + evalValue: "route legacy/policy", + costValue: "not reported", + }); + }); +}); From 7a637fa2f241e450458c4847dfd813d1f07dd40f Mon Sep 17 00:00:00 2001 From: homen Date: Tue, 14 Jul 2026 22:37:53 -0700 Subject: [PATCH 03/12] fix(nodeagent): harden SpreadsheetBench V2 execution Covers scripts, src runtime, and tests for traced workbook execution and official projection. --- .../spreadsheetbench-official-v2-project.ts | 1055 ++++++++++++++++- src/eval/spreadsheetBenchNodeAgentBridge.ts | 27 +- src/nodeagent/core/formulaEngine.ts | 99 +- src/nodeagent/core/runtime.ts | 37 +- .../spreadsheet/workbookTaskIntelligence.ts | 292 ++++- tests/agentRuntime.test.ts | 92 ++ tests/formulaEngine.test.ts | 39 +- tests/spreadsheetBenchNodeAgentBridge.test.ts | 238 +++- .../spreadsheetBenchOfficialV2Project.test.ts | 1046 ++++++++++++++++ tests/workbookTaskIntelligence.test.ts | 248 +++- 10 files changed, 3090 insertions(+), 83 deletions(-) create mode 100644 tests/spreadsheetBenchOfficialV2Project.test.ts diff --git a/scripts/spreadsheetbench-official-v2-project.ts b/scripts/spreadsheetbench-official-v2-project.ts index aa5bb747..4df29812 100644 --- a/scripts/spreadsheetbench-official-v2-project.ts +++ b/scripts/spreadsheetbench-official-v2-project.ts @@ -1,30 +1,121 @@ import { createHash } from "node:crypto"; -import { cpSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; -import { dirname, join, relative, resolve } from "node:path"; +import { + cpSync, + existsSync, + mkdirSync, + readFileSync, + realpathSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; +import { isDeepStrictEqual } from "node:util"; +import { SPREADSHEETBENCH_NODEAGENT_BRIDGE_SCHEMA } from "../src/eval/spreadsheetBenchNodeAgentBridge"; +import { stableTraceHash } from "../src/nodeagent/traces"; type RunReport = { schema?: number; mode?: string; outputRoot?: string; taskCount?: number; + harness?: { + toolPolicy?: string; + evaluatorAccess?: string; + }; results?: Array<{ taskId?: string; category?: string; mode?: string; candidateWorkbook?: string; model?: { name?: string; calls?: number }; + sidecarEvidence?: { + nodeAgentReceipt?: FileEvidence; + nodeAgentTrace?: FileEvidence; + }; }>; }; +type FileEvidence = { path?: string; sha256?: string; bytes?: number }; + +type JsonObject = Record; + +type LoadedJsonEvidence = + | { ok: true; evidence: Required; value: JsonObject } + | { ok: false; error: string }; + +type ValidatedNodeAgentEvidence = { + receiptEvidence: Required; + traceEvidence: Required; +}; + type DatasetTask = { id?: string | number }; -const CATEGORIES = ["Debugging", "Financial_Model", "Template", "Visualization"] as const; +const CATEGORIES = [ + "Debugging", + "Financial_Model", + "Template", + "Visualization", +] as const; const EXPECTED_COUNTS: Record<(typeof CATEGORIES)[number], number> = { Debugging: 100, Financial_Model: 100, Template: 97, Visualization: 24, }; +const NODEAGENT_RECEIPT_SCHEMA = SPREADSHEETBENCH_NODEAGENT_BRIDGE_SCHEMA; +const NODEAGENT_TRACE_SCHEMA = "nodeagent.trace.v1"; +const NODEAGENT_RECEIPT_FILE = "nodeagent-workbook-receipt.json"; +const NODEAGENT_TRACE_FILE = "nodeagent-workbook-trace.json"; +const NODEAGENT_TOOL_POLICY = "agent_dir_only_until_candidate"; +const NODEAGENT_EVALUATOR_ACCESS = "after_candidate_emit_only"; +const NODEAGENT_STAGES = [ + "inspect", + "plan", + "preflight", + "write", + "verify", + "repair", +] as const; +const NODEAGENT_STAGE_STATUSES = new Set([ + "completed", + "needs_repair", + "blocked", + "failed", + "skipped", +]); +const NODEAGENT_OUTCOME_STATUSES = new Set([ + "completed", + "needs_repair", + "blocked", + "failed", +]); +const NODEAGENT_STOP_REASONS = new Set([ + "done", + "step_budget", + "time_budget", + "spend_budget", + "error", +]); +const NODEAGENT_FRAME_STATUSES = new Set([ + "pending", + "running", + "completed", + "blocked", + "skipped", + "failed", +]); +const NODEAGENT_TRACE_FINAL_STATUSES = new Set([ + "completed", + "failed", + "needs_review", + "cancelled", +]); +const NODEAGENT_MUTATION_TOOLS = new Set([ + "write_locked_cell", + "write_locked_cells", + "execute_verified_workbook_plan", +]); const args = process.argv.slice(2); const reportPath = resolve(requiredOption("--report")); @@ -35,11 +126,28 @@ const outputsRoot = resolve(requiredOption("--outputs-root")); const receiptOut = resolve(requiredOption("--receipt-out")); const clean = args.includes("--clean"); const report = readJson(reportPath); +const allowedModes = new Set(["model-edit-plan", "nodeagent-workbook"]); -if (report.schema !== 1) throw new Error(`run report schema must be 1, got ${String(report.schema)}`); -if (report.mode !== "model-edit-plan") throw new Error(`V2 official projection requires model-edit-plan, got ${String(report.mode)}`); +if (report.schema !== 1) + throw new Error(`run report schema must be 1, got ${String(report.schema)}`); +if (!report.mode || !allowedModes.has(report.mode)) { + throw new Error( + `V2 official projection requires model-edit-plan or nodeagent-workbook, got ${String(report.mode)}`, + ); +} if (report.taskCount !== 321 || report.results?.length !== 321) { - throw new Error(`V2 official projection requires 321 results, got taskCount=${String(report.taskCount)} results=${report.results?.length ?? 0}`); + throw new Error( + `V2 official projection requires 321 results, got taskCount=${String(report.taskCount)} results=${report.results?.length ?? 0}`, + ); +} +if ( + report.mode === "nodeagent-workbook" && + (report.harness?.toolPolicy !== NODEAGENT_TOOL_POLICY || + report.harness.evaluatorAccess !== NODEAGENT_EVALUATOR_ACCESS) +) { + throw new Error( + `nodeagent-workbook report harness must declare toolPolicy=${NODEAGENT_TOOL_POLICY} and evaluatorAccess=${NODEAGENT_EVALUATOR_ACCESS}`, + ); } const upstreamDataRoot = join(upstreamRepo, "data"); @@ -54,19 +162,27 @@ const datasetIds = new Map>(); for (const category of CATEGORIES) { const sourceCategory = join(datasetRoot, category); const datasetPath = join(sourceCategory, "dataset.json"); - if (!existsSync(datasetPath)) throw new Error(`dataset is missing: ${datasetPath}`); + if (!existsSync(datasetPath)) + throw new Error(`dataset is missing: ${datasetPath}`); const tasks = readJson(datasetPath); const ids = new Set(tasks.map((task) => String(task.id ?? ""))); if (ids.has("") || ids.size !== EXPECTED_COUNTS[category]) { - throw new Error(`${category} dataset must contain ${EXPECTED_COUNTS[category]} unique IDs, got ${ids.size}`); + throw new Error( + `${category} dataset must contain ${EXPECTED_COUNTS[category]} unique IDs, got ${ids.size}`, + ); } datasetIds.set(category, ids); mkdirSync(upstreamDataRoot, { recursive: true }); - cpSync(sourceCategory, join(upstreamDataRoot, category), { recursive: true, force: true }); + cpSync(sourceCategory, join(upstreamDataRoot, category), { + recursive: true, + force: true, + }); } const seen = new Set(); -const categoryCounts = Object.fromEntries(CATEGORIES.map((category) => [category, 0])) as Record; +const categoryCounts = Object.fromEntries( + CATEGORIES.map((category) => [category, 0]), +) as Record; const cases: Array<{ taskId: string; category: string; @@ -77,18 +193,26 @@ const cases: Array<{ sourceSha256: string; output: string; outputSha256: string; + nodeAgentReceipt?: FileEvidence; + nodeAgentTrace?: FileEvidence; }> = []; const errors: string[] = []; for (const result of report.results) { const taskId = result.taskId ?? ""; const slash = taskId.indexOf("/"); - const category = (result.category || taskId.slice(0, slash)) as (typeof CATEGORIES)[number]; + const category = taskId.slice(0, slash) as (typeof CATEGORIES)[number]; const id = slash >= 0 ? taskId.slice(slash + 1) : ""; if (!CATEGORIES.includes(category) || !id) { errors.push(`invalid task identity: ${taskId}`); continue; } + if (result.category !== undefined && result.category !== category) { + errors.push( + `task category conflicts with taskId: ${taskId} declares ${result.category}`, + ); + continue; + } if (seen.has(taskId)) { errors.push(`duplicate task: ${taskId}`); continue; @@ -98,8 +222,8 @@ for (const result of report.results) { errors.push(`task is absent from upstream dataset: ${taskId}`); continue; } - if ((result.mode ?? report.mode) !== "model-edit-plan") { - errors.push(`task is not model-generated: ${taskId}`); + if ((result.mode ?? report.mode) !== report.mode) { + errors.push(`task mode does not match report mode: ${taskId}`); continue; } const candidateWorkbook = result.candidateWorkbook; @@ -108,16 +232,35 @@ for (const result of report.results) { continue; } const source = resolve(runRoot, candidateWorkbook); - if (!existsSync(source)) { + if (!isExistingFileWithin(runRoot, source)) { errors.push(`candidate workbook missing on disk: ${taskId} -> ${source}`); continue; } + const sourceSha256 = sha256File(source); + let nodeAgentEvidence: ValidatedNodeAgentEvidence | undefined; + if (report.mode === "nodeagent-workbook") { + const validation = validateNodeAgentEvidence({ + runRoot, + taskId, + category, + candidatePath: source, + candidateSha256: sourceSha256, + reportModel: result.model, + receiptEvidence: result.sidecarEvidence?.nodeAgentReceipt, + traceEvidence: result.sidecarEvidence?.nodeAgentTrace, + }); + if (!validation.ok) { + errors.push(`${validation.error}: ${taskId}`); + continue; + } + nodeAgentEvidence = validation.value; + } const output = join(outputsRoot, category, `${id}_output.xlsx`); mkdirSync(dirname(output), { recursive: true }); cpSync(source, output, { force: true }); - const sourceSha256 = sha256File(source); const outputSha256 = sha256File(output); - if (sourceSha256 !== outputSha256) errors.push(`copy hash mismatch: ${taskId}`); + if (sourceSha256 !== outputSha256) + errors.push(`copy hash mismatch: ${taskId}`); categoryCounts[category] += 1; cases.push({ taskId, @@ -129,24 +272,39 @@ for (const result of report.results) { sourceSha256, output: rel(output), outputSha256, + ...(nodeAgentEvidence + ? { + nodeAgentReceipt: nodeAgentEvidence.receiptEvidence, + nodeAgentTrace: nodeAgentEvidence.traceEvidence, + } + : {}), }); } for (const category of CATEGORIES) { if (categoryCounts[category] !== EXPECTED_COUNTS[category]) { - errors.push(`${category} projection expected ${EXPECTED_COUNTS[category]} outputs, got ${categoryCounts[category]}`); + errors.push( + `${category} projection expected ${EXPECTED_COUNTS[category]} outputs, got ${categoryCounts[category]}`, + ); } } -if (seen.size !== 321) errors.push(`projection expected 321 unique tasks, got ${seen.size}`); +if (seen.size !== 321) + errors.push(`projection expected 321 unique tasks, got ${seen.size}`); const evaluatorPath = join(upstreamRepo, "evaluation", "evaluation.py"); -const visualEvaluatorPath = join(upstreamRepo, "evaluation", "run_visual_vlm_checklist_eval.py"); -if (!existsSync(evaluatorPath) || !existsSync(visualEvaluatorPath)) throw new Error("upstream V2 evaluators are missing"); +const visualEvaluatorPath = join( + upstreamRepo, + "evaluation", + "run_visual_vlm_checklist_eval.py", +); +if (!existsSync(evaluatorPath) || !existsSync(visualEvaluatorPath)) + throw new Error("upstream V2 evaluators are missing"); cases.sort((a, b) => a.taskId.localeCompare(b.taskId)); const caseManifestSha256 = sha256(JSON.stringify(cases)); writeJson(receiptOut, { schema: 1, track: "spreadsheetbench-v2", + harnessMode: report.mode, generatedAt: new Date().toISOString(), report: rel(reportPath), reportSha256: sha256File(reportPath), @@ -167,7 +325,9 @@ writeJson(receiptOut, { errors, cases, }); -console.log(`projected ${cases.length}/321 V2 model outputs (${errors.length} errors)`); +console.log( + `projected ${cases.length}/321 V2 ${report.mode} outputs (${errors.length} errors)`, +); console.log(`wrote ${rel(receiptOut)}`); if (errors.length) process.exitCode = 1; @@ -188,6 +348,859 @@ function sha256(value: string): string { return createHash("sha256").update(value).digest("hex"); } +function validateNodeAgentEvidence(args: { + runRoot: string; + taskId: string; + category: (typeof CATEGORIES)[number]; + candidatePath: string; + candidateSha256: string; + reportModel?: { name?: string; calls?: number }; + receiptEvidence?: FileEvidence; + traceEvidence?: FileEvidence; +}): + | { ok: true; value: ValidatedNodeAgentEvidence } + | { ok: false; error: string } { + const receiptFile = loadJsonEvidence({ + root: args.runRoot, + candidatePath: args.candidatePath, + expectedBasename: NODEAGENT_RECEIPT_FILE, + label: "NodeAgent receipt", + evidence: args.receiptEvidence, + }); + if (!receiptFile.ok) return receiptFile; + const traceFile = loadJsonEvidence({ + root: args.runRoot, + candidatePath: args.candidatePath, + expectedBasename: NODEAGENT_TRACE_FILE, + label: "NodeAgent trace", + evidence: args.traceEvidence, + }); + if (!traceFile.ok) return traceFile; + + const receipt = receiptFile.value; + if (receipt.schema !== NODEAGENT_RECEIPT_SCHEMA) + return invalid("NodeAgent receipt schema mismatch"); + if (receipt.taskId !== args.taskId) + return invalid("NodeAgent receipt taskId mismatch"); + if (receipt.track !== "spreadsheetbench-v2") + return invalid("NodeAgent receipt track mismatch"); + if (receipt.category !== undefined && receipt.category !== args.category) + return invalid("NodeAgent receipt category mismatch"); + const traceId = nonemptyString(receipt.traceId); + if (!traceId) return invalid("NodeAgent receipt traceId is missing"); + if (!nonemptyString(receipt.candidateWorkbookPath)) + return invalid("NodeAgent receipt candidate path is missing"); + if (resolve(receipt.candidateWorkbookPath as string) !== resolve(args.candidatePath)) + return invalid("NodeAgent receipt candidate path mismatch"); + if ( + !isSha256(receipt.candidateWorkbookSha256) || + receipt.candidateWorkbookSha256.toLowerCase() !== args.candidateSha256 + ) { + return invalid("NodeAgent receipt candidate workbook hash mismatch"); + } + + const isolation = asObject(receipt.isolation); + if ( + isolation?.boundary !== "agent_visible_files_only" || + !nonemptyString(isolation.agentRoot) || + !nonemptyStringArray(isolation.openedAgentFiles) || + isolation.evaluatorMetadataAccess !== "none" || + isolation.evaluatorFileReadCount !== 0 || + isolation.candidateEmittedBeforeEvaluatorAccess !== true + ) { + return invalid( + "NodeAgent receipt isolation/evaluator-access contract mismatch", + ); + } + + const reportModelName = nonemptyString(args.reportModel?.name); + const reportModelCalls = args.reportModel?.calls; + if ( + !reportModelName || + !Number.isInteger(reportModelCalls) || + (reportModelCalls ?? 0) < 1 + ) { + return invalid("NodeAgent report model identity is invalid"); + } + const receiptModel = asObject(receipt.model); + const receiptModelName = nonemptyString(receiptModel?.name); + const receiptModelCalls = receiptModel?.calls; + if (!receiptModelName || receiptModelName !== reportModelName) + return invalid("NodeAgent receipt model name mismatch"); + if ( + !Number.isInteger(receiptModelCalls) || + receiptModelCalls !== reportModelCalls || + (receiptModelCalls as number) < 1 + ) { + return invalid("NodeAgent receipt model calls mismatch"); + } + + const traceError = validateNodeAgentTrace(traceFile.value, { + taskId: args.taskId, + traceId, + candidatePath: receipt.candidateWorkbookPath as string, + candidateSha256: args.candidateSha256, + }); + if (traceError) return invalid(traceError); + const embeddedTrace = asObject(receipt.trace); + if (!embeddedTrace) + return invalid("NodeAgent receipt embedded trace is missing"); + if (!isDeepStrictEqual(embeddedTrace, traceFile.value)) { + return invalid( + "NodeAgent receipt embedded trace does not match trace sidecar", + ); + } + const consistencyError = validateNodeAgentExecutionConsistency( + receipt, + traceFile.value, + traceId, + ); + if (consistencyError) return invalid(consistencyError); + + return { + ok: true, + value: { + receiptEvidence: receiptFile.evidence, + traceEvidence: traceFile.evidence, + }, + }; +} + +function isPathWithin(root: string, path: string): boolean { + const relativePath = relative(root, path); + return ( + Boolean(relativePath) && + relativePath !== ".." && + !relativePath.startsWith(`..${sep}`) && + !isAbsolute(relativePath) + ); +} + +function isExistingFileWithin(root: string, path: string): boolean { + try { + return ( + existsSync(path) && + statSync(path).isFile() && + isPathWithin(realpathSync(root), realpathSync(path)) + ); + } catch { + return false; + } +} + +function loadJsonEvidence(args: { + root: string; + candidatePath: string; + expectedBasename: string; + label: string; + evidence?: FileEvidence; +}): LoadedJsonEvidence { + const evidencePath = args.evidence?.path; + const declaredSha256 = args.evidence?.sha256; + const declaredBytes = args.evidence?.bytes; + if ( + !nonemptyString(evidencePath) || + isAbsolute(evidencePath) || + !isSha256(declaredSha256) || + !Number.isInteger(declaredBytes) || + (declaredBytes ?? 0) <= 0 + ) { + return invalid(`${args.label} evidence metadata is invalid`); + } + const path = resolve(args.root, evidencePath); + const expectedPath = join(dirname(args.candidatePath), args.expectedBasename); + if (relative(path, expectedPath) !== "") + return invalid(`${args.label} path is not canonical for the candidate`); + if (!isExistingFileWithin(args.root, path)) + return invalid(`${args.label} file is missing or outside the run root`); + + const content = readFileSync(path); + const actualSha256 = createHash("sha256").update(content).digest("hex"); + if ( + content.byteLength !== declaredBytes || + actualSha256 !== declaredSha256.toLowerCase() + ) { + return invalid(`${args.label} evidence hash or byte count mismatch`); + } + let value: unknown; + try { + value = JSON.parse(content.toString("utf8")); + } catch { + return invalid(`${args.label} is not valid JSON`); + } + const object = asObject(value); + if (!object) return invalid(`${args.label} JSON root must be an object`); + return { + ok: true, + evidence: { + path: evidencePath, + sha256: declaredSha256, + bytes: declaredBytes as number, + }, + value: object, + }; +} + +function validateNodeAgentTrace( + trace: JsonObject, + expected: { + taskId: string; + traceId: string; + candidatePath: string; + candidateSha256: string; + }, +): string | undefined { + if (trace.schema !== NODEAGENT_TRACE_SCHEMA) + return "NodeAgent trace schema mismatch"; + if (trace.traceId !== expected.traceId) return "NodeAgent traceId mismatch"; + if (asObject(trace.trigger)?.kind !== "benchmark") + return "NodeAgent trace trigger mismatch"; + + const evalReceipt = asObject(trace.eval); + if (evalReceipt?.benchmarkCaseId !== expected.taskId) + return "NodeAgent trace benchmark case mismatch"; + const proofRefError = candidateArtifactRefError( + evalReceipt?.proofArtifacts, + expected, + "NodeAgent trace candidate proof hash mismatch", + ); + if (proofRefError) return proofRefError; + + const finalReceipt = asObject(trace.final); + if ( + !finalReceipt || + !["completed", "failed", "needs_review", "cancelled"].includes( + String(finalReceipt.status), + ) + ) { + return "NodeAgent trace final receipt is invalid"; + } + return candidateArtifactRefError( + finalReceipt.outputArtifactRefs, + expected, + "NodeAgent trace final artifact hash mismatch", + ); +} + +function validateNodeAgentExecutionConsistency( + receipt: JsonObject, + trace: JsonObject, + traceId: string, +): string | undefined { + const outcome = asObject(receipt.outcome); + const outcomeStatus = outcome?.status; + const mutatingTask = outcome?.mutatingTask; + const changedCellCount = outcome?.changedCellCount; + const finalVerificationStatus = outcome?.finalVerificationStatus; + if ( + !outcome || + !NODEAGENT_OUTCOME_STATUSES.has(String(outcomeStatus)) || + typeof mutatingTask !== "boolean" || + !isNonnegativeInteger(changedCellCount) || + !["passed", "needs_repair", "missing"].includes( + String(finalVerificationStatus), + ) + ) { + return "NodeAgent receipt outcome is invalid"; + } + const frame = asObject(receipt.frame); + const frameStatus = frame?.status; + const agentResult = asObject(frame?.agentResult); + const stopReason = agentResult?.stopReason; + const frameUsage = asObject(agentResult?.usage); + const frameTrace = agentResult?.trace; + if ( + !frame || + !NODEAGENT_FRAME_STATUSES.has(String(frameStatus)) || + !agentResult || + !NODEAGENT_STOP_REASONS.has(String(stopReason)) || + !frameUsage || + !Array.isArray(frameTrace) || + frameTrace.length === 0 + ) { + return "NodeAgent frame execution receipt is invalid"; + } + + const receiptModel = asObject(receipt.model); + const receiptUsage = asObject(receiptModel?.usage); + if ( + !receiptUsage || + !sameNonnegativeInteger(receiptModel?.calls, frameUsage.modelCalls) || + !sameNonnegativeInteger(receiptUsage.inputTokens, frameUsage.inputTokens) || + !sameNonnegativeInteger(receiptUsage.outputTokens, frameUsage.outputTokens) || + (receiptUsage.cachedInputTokens !== undefined && + !sameNonnegativeInteger( + receiptUsage.cachedInputTokens, + frameUsage.cachedInputTokens, + )) + ) { + return "NodeAgent frame/model usage mismatch"; + } + + const frameEvents: JsonObject[] = []; + for (const value of frameTrace) { + const event = asObject(value); + if ( + !event || + !isNonnegativeInteger(event.step) || + !nonemptyString(event.tool) || + typeof event.ms !== "number" || + !Number.isFinite(event.ms) || + event.ms < 0 + ) { + return "NodeAgent frame trace event is invalid"; + } + frameEvents.push(event); + } + + const traceSteps = trace.steps; + if (!Array.isArray(traceSteps) || traceSteps.length !== frameEvents.length) { + return "NodeAgent trace steps do not match frame trace"; + } + for (let index = 0; index < frameEvents.length; index += 1) { + const event = frameEvents[index]; + const step = asObject(traceSteps[index]); + const tool = asObject(step?.tool); + const argsHash = stableTraceHash(event.args); + const resultHash = stableTraceHash(event.result); + const expectedToolStatus = traceEventFailed(event.result) ? "failed" : "ok"; + if ( + !step || + step.traceId !== traceId || + !nonemptyString(step.stepId) || + !tool || + tool.name !== event.tool || + tool.argsHash !== argsHash || + tool.resultHash !== resultHash || + tool.status !== expectedToolStatus || + !traceRefArrayContainsHash(step.inputRefs, argsHash) || + !traceRefArrayContainsHash(step.outputRefs, resultHash) + ) { + return "NodeAgent trace step does not bind its frame event"; + } + } + + const stages = asObject(receipt.stages); + if (!stages) return "NodeAgent stage receipts are missing"; + const stageReceipts = new Map(); + for (const stageName of NODEAGENT_STAGES) { + const stage = asObject(stages[stageName]); + const events = stage?.events; + if ( + !stage || + stage.traceId !== traceId || + stage.stage !== stageName || + !NODEAGENT_STAGE_STATUSES.has(String(stage.status)) || + !isNonnegativeInteger(stage.attempts) || + !Array.isArray(events) || + stage.attempts !== events.length || + (stage.operationCount !== undefined && + !isNonnegativeInteger(stage.operationCount)) || + !nonemptyString(stage.summary) + ) { + return `NodeAgent ${stageName} stage receipt is invalid`; + } + if (stage.status === "completed" && events.length === 0) { + return `NodeAgent completed ${stageName} stage has no events`; + } + if (stage.status === "skipped" && events.length !== 0) { + return `NodeAgent skipped ${stageName} stage has events`; + } + for (const value of events) { + const eventRef = asObject(value); + const eventIndex = eventRef?.eventIndex; + if ( + !eventRef || + eventRef.traceId !== traceId || + !isNonnegativeInteger(eventIndex) || + (eventIndex as number) >= frameEvents.length + ) { + return `NodeAgent ${stageName} stage event reference is invalid`; + } + const frameEvent = frameEvents[eventIndex as number]; + if ( + eventRef.step !== frameEvent.step || + eventRef.tool !== frameEvent.tool || + eventRef.argsHash !== stableTraceHash(frameEvent.args) || + eventRef.resultHash !== stableTraceHash(frameEvent.result) + ) { + return `NodeAgent ${stageName} stage event does not bind its frame event`; + } + } + stageReceipts.set(stageName, stage); + } + const derivedStages = deriveNodeAgentStages(frameEvents); + for (const stageName of NODEAGENT_STAGES) { + const stage = stageReceipts.get(stageName)!; + const events = stage.events as unknown[]; + const actualEventIndexes = events.map( + (value) => asObject(value)!.eventIndex as number, + ); + const expected = derivedStages[stageName]; + if (!isDeepStrictEqual(actualEventIndexes, expected.eventIndexes)) { + return `NodeAgent ${stageName} stage events do not match frame semantics`; + } + if (stage.status !== expected.status) { + return `NodeAgent ${stageName} stage status does not match frame events`; + } + if (stage.operationCount !== expected.operationCount) { + return `NodeAgent ${stageName} operation count does not match frame events`; + } + } + + const traceEvidence = trace.evidence; + if (!Array.isArray(traceEvidence) || traceEvidence.length === 0) { + return "NodeAgent trace evidence is missing"; + } + for (const value of traceEvidence) { + const evidence = asObject(value); + if ( + !evidence || + evidence.traceId !== traceId || + !nonemptyString(evidence.receiptId) || + !nonemptyString(evidence.label) || + !["verified", "needs_review", "rejected"].includes( + String(evidence.status), + ) || + !Array.isArray(evidence.sourceRefs) || + !Array.isArray(evidence.artifactRefs) + ) { + return "NodeAgent trace evidence receipt is invalid"; + } + } + + const mutationEvents = frameEvents.filter((event) => + NODEAGENT_MUTATION_TOOLS.has(String(event.tool)), + ); + const mutations = trace.mutations; + if (!Array.isArray(mutations) || mutations.length !== mutationEvents.length) { + return "NodeAgent trace mutations do not match frame writes"; + } + let committedMutationCount = 0; + let committedTargetOccurrences = 0; + for (let index = 0; index < mutationEvents.length; index += 1) { + const event = mutationEvents[index]; + const mutation = asObject(mutations[index]); + const targets = mutation?.targetRefs; + const expectedStatus = traceMutationStatus(event.result); + if ( + !mutation || + mutation.traceId !== traceId || + !nonemptyString(mutation.receiptId) || + mutation.payloadHash !== stableTraceHash(event.args) || + mutation.status !== expectedStatus || + !Array.isArray(targets) + ) { + return "NodeAgent mutation receipt does not bind its frame write"; + } + if (mutation.status === "committed" && targets.length === 0) { + return "NodeAgent committed mutation has no targets"; + } + if (mutation.status === "committed") { + committedMutationCount += 1; + committedTargetOccurrences += targets.length; + } + for (const value of targets) { + const target = asObject(value); + if (!target || !nonemptyString(target.refId) || !nonemptyString(target.kind)) { + return "NodeAgent mutation target is invalid"; + } + } + } + const observedCommittedMutation = committedMutationCount > 0; + if (observedCommittedMutation !== ((changedCellCount as number) > 0)) { + return "NodeAgent changed cell count does not match committed mutations"; + } + + const traceFinal = asObject(trace.final); + const expectedFinalStatus = outcomeStatus === "completed" + ? "completed" + : outcomeStatus === "failed" + ? "failed" + : "needs_review"; + if ( + !traceFinal || + !NODEAGENT_TRACE_FINAL_STATUSES.has(String(traceFinal.status)) || + traceFinal.status !== expectedFinalStatus + ) { + return "NodeAgent trace final status does not match outcome"; + } + if (frameStatus === "completed" && stopReason !== "done") { + return "NodeAgent frame completion does not match stop reason"; + } + if (stopReason !== "done" && frameStatus !== "blocked") { + return "NodeAgent terminal frame stop is not blocked"; + } + if ( + outcomeStatus === "completed" && + stageReceipts.get("inspect")?.status !== "completed" + ) { + return "NodeAgent completed outcome lacks completed inspect stage"; + } + if ( + outcomeStatus === "failed" && + stopReason !== "error" && + !nonemptyString(frame.runtimeError) + ) { + return "NodeAgent failed outcome does not match frame failure"; + } + if ( + outcomeStatus === "blocked" && + frameStatus !== "blocked" && + stopReason === "done" + ) { + return "NodeAgent blocked outcome does not match frame stop"; + } + + const completedMutation = outcomeStatus === "completed"; + if (completedMutation) { + if ( + !observedCommittedMutation || + (changedCellCount as number) < 1 || + finalVerificationStatus !== "passed" + ) { + return "NodeAgent completed mutation lacks changed cells or verification"; + } + for (const stageName of [ + "inspect", + "plan", + "preflight", + "write", + "verify", + ] as const) { + if (stageReceipts.get(stageName)?.status !== "completed") { + return `NodeAgent completed mutation lacks completed ${stageName} stage`; + } + } + for (const stageName of ["plan", "preflight", "write", "verify"] as const) { + const operationCount = stageReceipts.get(stageName)?.operationCount; + if (!isPositiveInteger(operationCount)) { + return `NodeAgent completed mutation lacks ${stageName} operations`; + } + } + if ( + (stageReceipts.get("write")?.operationCount as number) < + (changedCellCount as number) + ) { + return "NodeAgent write operation count is below changed cell count"; + } + if (committedTargetOccurrences < (changedCellCount as number)) { + return "NodeAgent committed mutation targets are below changed cell count"; + } + } + if ( + finalVerificationStatus === "passed" && + stageReceipts.get("verify")?.status !== "completed" + ) { + return "NodeAgent passed verification lacks completed verify stage"; + } + + const recalculation = asObject(receipt.recalculation); + if ( + !recalculation || + recalculation.engine !== "nodeagent-formula-engine" || + !isNonnegativeInteger(recalculation.attemptedFormulaCount) || + !isNonnegativeInteger(recalculation.refreshedFormulaCount) || + !isNonnegativeInteger(recalculation.unresolvedFormulaCount) || + !Array.isArray(recalculation.unresolved) || + recalculation.unresolvedFormulaCount !== recalculation.unresolved.length + ) { + return "NodeAgent recalculation receipt is invalid"; + } + if ( + outcomeStatus === "completed" && + recalculation.unresolvedFormulaCount !== 0 + ) { + return "NodeAgent completed outcome has unresolved formulas"; + } + return undefined; +} + +function candidateArtifactRefError( + value: unknown, + expected: { candidatePath: string; candidateSha256: string }, + message: string, +): string | undefined { + if (!Array.isArray(value) || value.length !== 1) return message; + const ref = asObject(value[0]); + return ref?.kind === "artifact" && + ref.refId === expected.candidatePath && + isSha256(ref.hash) && + ref.hash.toLowerCase() === expected.candidateSha256 + ? undefined + : message; +} + +function asObject(value: unknown): JsonObject | undefined { + return value !== null && typeof value === "object" && !Array.isArray(value) + ? (value as JsonObject) + : undefined; +} + +function isNonnegativeInteger(value: unknown): value is number { + return Number.isInteger(value) && (value as number) >= 0; +} + +function isPositiveInteger(value: unknown): value is number { + return Number.isInteger(value) && (value as number) > 0; +} + +function sameNonnegativeInteger(left: unknown, right: unknown): boolean { + return ( + isNonnegativeInteger(left) && + isNonnegativeInteger(right) && + left === right + ); +} + +function traceRefArrayContainsHash(value: unknown, hash: string): boolean { + return ( + Array.isArray(value) && + value.some((entry) => asObject(entry)?.hash === hash) + ); +} + +function traceEventFailed(result: unknown): boolean { + const record = asObject(result); + return Boolean(record && (record.ok === false || typeof record.error === "string")); +} + +function traceMutationStatus( + result: unknown, +): "proposed" | "committed" | "skipped" | "conflict" | "pending_approval" { + const record = asObject(result); + if (record?.pendingApproval === true) return "pending_approval"; + if (record?.conflict === true) return "conflict"; + if (record?.skipped === true) return "skipped"; + if (record?.ok === true) return "committed"; + return "conflict"; +} + +type DerivedNodeAgentStage = { + status: string; + eventIndexes: number[]; + operationCount?: number; +}; + +function deriveNodeAgentStages( + frameEvents: JsonObject[], +): Record<(typeof NODEAGENT_STAGES)[number], DerivedNodeAgentStage> { + const indexed = frameEvents.map((event, eventIndex) => ({ event, eventIndex })); + const inspect = indexed.filter(({ event }) => event.tool === "inspect_workbook"); + const composite = indexed.filter( + ({ event }) => event.tool === "execute_verified_workbook_plan", + ); + const verifications = indexed.filter( + ({ event }) => event.tool === "verify_workbook", + ); + const explicitPreflights = verifications.filter( + ({ event }) => nodeAgentVerificationPhase(event) === "preflight", + ); + const explicitPostWrites = verifications.filter( + ({ event }) => nodeAgentVerificationPhase(event) === "post_write", + ); + const compositePreflights = composite.filter( + ({ event }) => + nodeAgentCompositePhaseStatus(event.result, "preflight") !== "missing", + ); + const compositePostWrites = composite.filter( + ({ event }) => + nodeAgentCompositePhaseStatus(event.result, "verify") !== "missing", + ); + const preflights = [...explicitPreflights, ...compositePreflights].sort( + (left, right) => left.eventIndex - right.eventIndex, + ); + const postWrites = [...explicitPostWrites, ...compositePostWrites].sort( + (left, right) => left.eventIndex - right.eventIndex, + ); + const writes = indexed.filter(({ event }) => + NODEAGENT_MUTATION_TOOLS.has(String(event.tool)), + ); + const failedVerifications = [ + ...explicitPreflights.filter( + ({ event }) => nodeAgentVerificationStatus(event.result) !== "passed", + ), + ...explicitPostWrites.filter( + ({ event }) => nodeAgentVerificationStatus(event.result) !== "passed", + ), + ...compositePreflights.filter( + ({ event }) => + nodeAgentCompositePhaseStatus(event.result, "preflight") !== "passed", + ), + ...compositePostWrites.filter( + ({ event }) => + nodeAgentCompositePhaseStatus(event.result, "verify") !== "passed", + ), + ].sort((left, right) => left.eventIndex - right.eventIndex); + const latestPreflight = preflights.at(-1)?.event; + const latestPostWrite = postWrites.at(-1)?.event; + const planOperationCount = + nodeAgentOperationCount(latestPreflight?.args) || + nodeAgentCompositeOperationCount(latestPreflight?.result); + const repairResolved = + failedVerifications.length > 0 && + nodeAgentEventVerificationStatus(latestPostWrite, "verify") === "passed" && + failedVerifications.some(({ eventIndex }) => + indexed.some( + (entry) => + entry.eventIndex > eventIndex && + (entry.event.tool === "verify_workbook" || + NODEAGENT_MUTATION_TOOLS.has(String(entry.event.tool))), + ), + ); + const inspectStatus = + inspect.length === 0 + ? "skipped" + : inspect.some(({ event }) => nodeAgentEventSucceeded(event.result)) + ? "completed" + : "failed"; + const preflightStatus = latestPreflight + ? nodeAgentEventVerificationStatus(latestPreflight, "preflight") === "passed" + ? "completed" + : "needs_repair" + : "skipped"; + const writeStatus = + writes.length === 0 + ? "skipped" + : nodeAgentEventSucceeded(writes.at(-1)!.event.result) + ? "completed" + : "blocked"; + const verifyStatus = latestPostWrite + ? nodeAgentEventVerificationStatus(latestPostWrite, "verify") === "passed" + ? "completed" + : "needs_repair" + : "skipped"; + return { + inspect: { + status: inspectStatus, + eventIndexes: inspect.map(({ eventIndex }) => eventIndex), + }, + plan: { + status: preflights.length ? preflightStatus : "skipped", + eventIndexes: preflights.map(({ eventIndex }) => eventIndex), + operationCount: planOperationCount, + }, + preflight: { + status: preflightStatus, + eventIndexes: preflights.map(({ eventIndex }) => eventIndex), + operationCount: planOperationCount, + }, + write: { + status: writeStatus, + eventIndexes: writes.map(({ eventIndex }) => eventIndex), + operationCount: writes.reduce( + (sum, { event }) => + sum + + (nodeAgentOperationCount(event.args) || + nodeAgentCompositeOperationCount(event.result)), + 0, + ), + }, + verify: { + status: verifyStatus, + eventIndexes: postWrites.map(({ eventIndex }) => eventIndex), + operationCount: + nodeAgentOperationCount(latestPostWrite?.args) || + nodeAgentCompositeOperationCount(latestPostWrite?.result), + }, + repair: { + status: + failedVerifications.length === 0 + ? "skipped" + : repairResolved + ? "completed" + : "needs_repair", + eventIndexes: failedVerifications.map(({ eventIndex }) => eventIndex), + }, + }; +} + +function nodeAgentVerificationPhase( + event: JsonObject, +): "preflight" | "post_write" { + const result = asObject(event.result); + if (result?.phase === "preflight") return "preflight"; + if (result?.phase === "post_write") return "post_write"; + return asObject(event.args)?.afterWrite === false ? "preflight" : "post_write"; +} + +function nodeAgentVerificationStatus( + result: unknown, +): "passed" | "needs_repair" | "missing" { + const status = asObject(result)?.status; + return status === "passed" + ? "passed" + : status === "needs_repair" + ? "needs_repair" + : "missing"; +} + +function nodeAgentCompositePhaseStatus( + result: unknown, + phase: "preflight" | "verify", +): "passed" | "needs_repair" | "missing" { + const receipt = asObject(asObject(asObject(result)?.phases)?.[phase]); + const status = receipt?.status; + if (status === "passed" || status === "completed") return "passed"; + if (typeof status === "string" && status !== "skipped") return "needs_repair"; + return "missing"; +} + +function nodeAgentEventVerificationStatus( + event: JsonObject | undefined, + phase: "preflight" | "verify", +): "passed" | "needs_repair" | "missing" { + if (!event) return "missing"; + return event.tool === "execute_verified_workbook_plan" + ? nodeAgentCompositePhaseStatus(event.result, phase) + : nodeAgentVerificationStatus(event.result); +} + +function nodeAgentEventSucceeded(result: unknown): boolean { + const record = asObject(result); + if (!record) return true; + if (record.pendingApproval === true || record.drafted === true) return true; + return record.ok !== false && typeof record.error !== "string"; +} + +function nodeAgentOperationCount(args: unknown): number { + const record = asObject(args); + if (!record) return 0; + if (Array.isArray(record.operations)) return record.operations.length; + if (Array.isArray(record.ops)) return record.ops.length; + if (Array.isArray(record.cells)) return record.cells.length; + return typeof record.elementId === "string" ? 1 : 0; +} + +function nodeAgentCompositeOperationCount(result: unknown): number { + const count = asObject(result)?.operationCount; + return typeof count === "number" && Number.isFinite(count) + ? Math.max(0, Math.trunc(count)) + : 0; +} + +function nonemptyString(value: unknown): string | undefined { + return typeof value === "string" && value.trim() === value && value.length > 0 + ? value + : undefined; +} + +function nonemptyStringArray(value: unknown): value is string[] { + return ( + Array.isArray(value) && + value.length > 0 && + value.every((item) => Boolean(nonemptyString(item))) + ); +} + +function isSha256(value: unknown): value is string { + return typeof value === "string" && /^[a-f0-9]{64}$/i.test(value); +} + +function invalid(error: string): { ok: false; error: string } { + return { ok: false, error }; +} + function gitCommit(repo: string): string { const head = readFileSync(join(repo, ".git", "HEAD"), "utf8").trim(); if (!head.startsWith("ref: ")) return head; diff --git a/src/eval/spreadsheetBenchNodeAgentBridge.ts b/src/eval/spreadsheetBenchNodeAgentBridge.ts index 42e49caa..419b5b9e 100644 --- a/src/eval/spreadsheetBenchNodeAgentBridge.ts +++ b/src/eval/spreadsheetBenchNodeAgentBridge.ts @@ -261,6 +261,7 @@ export async function runSpreadsheetBenchNodeAgentBridge( room.changedCellCount(), bridgeWorkbookRepairContract(room).requiredRepairs.length, workflowController.pendingVerificationCount(), + recalculation.unresolvedFormulaCount, ); const trace = buildBridgeTrace({ traceId, @@ -866,7 +867,10 @@ class BridgeWorkbookWorkflowController { return this.withExecution(tool, async (args, rt) => { const requested = asRecord(args) ?? {}; const repairContract = bridgeWorkbookRepairContract(rt); - const artifactId = bridgePreferredRepairArtifact(repairContract, requested.artifactId); + const preferredArtifactId = bridgePreferredRepairArtifact(repairContract, requested.artifactId); + const artifactId = preferredArtifactId + ? await bridgeResolvedArtifactId(rt, preferredArtifactId, this.activeArtifactId) + : undefined; const result = await tool.execute({ ...requested, instruction: this.taskInstruction, @@ -1551,15 +1555,20 @@ function bridgeOutcome( changedCellCount: number, remainingHighConfidenceRepairs: number, pendingVerificationCount: number, + unresolvedFormulaCount: number, ): SpreadsheetBenchNodeAgentBridgeReceipt["outcome"] { - const finalVerificationStatus = pendingVerificationCount > 0 + const finalVerificationStatus = unresolvedFormulaCount > 0 + ? "needs_repair" as const + : pendingVerificationCount > 0 ? "missing" as const : stages.verify.status === "completed" ? "passed" as const : stages.verify.status === "needs_repair" ? "needs_repair" as const : "missing" as const; let status: SpreadsheetBenchNodeAgentBridgeReceipt["outcome"]["status"]; - const deterministicProofComplete = stages.inspect.status === "completed" - && (!mutatingTask || ( + const observedMutation = mutatingTask || changedCellCount > 0; + const deterministicProofComplete = unresolvedFormulaCount === 0 + && stages.inspect.status === "completed" + && (!observedMutation || ( changedCellCount > 0 && stages.plan.status === "completed" && stages.preflight.status === "completed" @@ -1570,15 +1579,21 @@ function bridgeOutcome( )); if (deterministicProofComplete) status = "completed"; else if (frameReceipt.runtimeError || frameReceipt.agentResult.stopReason === "error") status = "failed"; + else if (unresolvedFormulaCount > 0) status = "needs_repair"; + else if (stages.preflight.status === "needs_repair" + || stages.verify.status === "needs_repair" + || remainingHighConfidenceRepairs > 0 + || pendingVerificationCount > 0) status = "needs_repair"; else if (frameReceipt.status === "blocked" || frameReceipt.agentResult.stopReason !== "done") status = "blocked"; else if (stages.inspect.status !== "completed") status = "needs_repair"; - else if (mutatingTask && ( + else if (observedMutation && ( stages.plan.status !== "completed" || stages.preflight.status !== "completed" || stages.write.status !== "completed" || stages.verify.status !== "completed" || remainingHighConfidenceRepairs > 0 || pendingVerificationCount > 0 + || unresolvedFormulaCount > 0 )) status = "needs_repair"; else status = "completed"; return { status, mutatingTask, changedCellCount, finalVerificationStatus }; @@ -1646,7 +1661,7 @@ function buildBridgeTrace(args: { artifactRefs: [candidateRef], fact: args.recalculation, verifier: args.recalculation.engine, - status: "verified", + status: args.recalculation.unresolvedFormulaCount === 0 ? "verified" : "needs_review", })); trace.mutations = args.frameReceipt.agentResult.trace.flatMap((event): MutationReceipt[] => { if (!MUTATION_TOOL_NAMES.has(event.tool)) return []; diff --git a/src/nodeagent/core/formulaEngine.ts b/src/nodeagent/core/formulaEngine.ts index 9dd2ffe5..37eb94be 100644 --- a/src/nodeagent/core/formulaEngine.ts +++ b/src/nodeagent/core/formulaEngine.ts @@ -8,8 +8,8 @@ * * Supports: numbers, "strings", TRUE/FALSE, trailing %, ( ); operators + - * / ^, unary +/-, * comparisons = <> < > <= >=; A1 refs (A1, $A$1) and ranges (A1:B3); and the finance-core - * functions SUM AVERAGE MIN MAX COUNT COUNTA IF AND OR NOT ROUND ROUNDUP ROUNDDOWN ABS SQRT - * CONCAT CONCATENATE. (Reuses the look-and-feel of the bench evaluator in spreadsheetBenchRunner.ts + * functions SUM AVERAGE MEDIAN MIN MAX COUNT COUNTA IF AND OR NOT ROUND ROUNDUP ROUNDDOWN ABS SQRT + * CONCAT CONCATENATE TEXT. (Reuses the look-and-feel of the bench evaluator in spreadsheetBenchRunner.ts * but is independently implemented for the browser; extend the function set as needed.) */ @@ -110,7 +110,7 @@ function tokenize(src: string): Tok[] { } const two = src.slice(i, i + 2); if (TWO_CHAR_OPS.has(two)) { toks.push({ t: "op", v: two }); i += 2; continue; } - if ("+-*/^(),:%<>=".includes(ch)) { toks.push({ t: "op", v: ch }); i++; continue; } + if ("+-*/^(),:%<>={}".includes(ch)) { toks.push({ t: "op", v: ch }); i++; continue; } throw new FormulaEvalError("#ERROR!"); // unrecognized character } return toks; @@ -121,6 +121,7 @@ type Node = | { k: "num"; v: number } | { k: "str"; v: string } | { k: "bool"; v: boolean } + | { k: "array"; values: Node[] } | { k: "ref"; v: string } | { k: "range"; a: string; b: string } | { k: "bin"; op: string; l: Node; r: Node } @@ -177,6 +178,12 @@ class Parser { const t = this.next(); if (t.t === "num") { const v = Number(t.v); if (!Number.isFinite(v)) throw new FormulaEvalError("#ERROR!"); return { k: "num", v }; } if (t.t === "str") return { k: "str", v: t.v }; + if (t.t === "op" && t.v === "{") { + const values: Node[] = [this.compare()]; + while (this.isOp(",")) { this.next(); values.push(this.compare()); } + this.eat("}"); + return { k: "array", values }; + } if (t.t === "op" && t.v === "(") { const e = this.compare(); this.eat(")"); return e; } if (t.t === "id") { const up = t.v.toUpperCase(); @@ -208,10 +215,10 @@ class Parser { /* evaluator */ const SUPPORTED = new Set([ - "SUM", "AVERAGE", "MIN", "MAX", "COUNT", "COUNTA", "IF", "AND", "OR", "NOT", + "SUM", "AVERAGE", "MEDIAN", "MIN", "MAX", "COUNT", "COUNTA", "IF", "AND", "OR", "NOT", "ROUND", "ROUNDUP", "ROUNDDOWN", "ABS", "SQRT", "CONCAT", "CONCATENATE", "SUMIF", "COUNTIF", "AVERAGEIF", "VLOOKUP", "INDEX", "MATCH", "IFERROR", - "MOD", "POWER", "LEN", "LEFT", "RIGHT", "MID", "TRIM", "UPPER", "LOWER", + "MOD", "POWER", "LEN", "LEFT", "RIGHT", "MID", "TRIM", "UPPER", "LOWER", "TEXT", ]); /** Coerce a value to a number for ARITHMETIC (blank -> 0, numeric string -> number, else #VALUE!). */ @@ -245,11 +252,14 @@ function aggNumber(v: CellValue): number | null { return null; // booleans & blanks ignored by SUM/AVERAGE/etc. } -function evalScalar(n: Node, R: CellResolver): CellValue { +type EvalValue = CellValue | CellValue[]; + +function evalValue(n: Node, R: CellResolver): EvalValue { switch (n.k) { case "num": return n.v; case "str": return n.v; case "bool": return n.v; + case "array": return n.values.map((value) => evalScalar(value, R)); case "ref": return R.getCell(normalizeRef(n.v)); case "range": throw new FormulaEvalError("#VALUE!"); // a bare range is not a scalar case "pct": return toNumber(evalScalar(n.e, R)) / 100; @@ -259,6 +269,12 @@ function evalScalar(n: Node, R: CellResolver): CellValue { } } +function evalScalar(n: Node, R: CellResolver): CellValue { + const value = evalValue(n, R); + if (Array.isArray(value)) throw new FormulaEvalError("#VALUE!"); + return value; +} + function evalBin(n: { op: string; l: Node; r: Node }, R: CellResolver): CellValue { const op = n.op; if (["=", "<>", "<", ">", "<=", ">="].includes(op)) { @@ -311,7 +327,8 @@ function truthy(v: CellValue): boolean { /** Flatten a function arg into a list of values (a range expands; everything else is one value). */ function argValues(n: Node, R: CellResolver): CellValue[] { if (n.k === "range") return expandRange(n.a, n.b).map((ref) => R.getCell(ref)); - return [evalScalar(n, R)]; + const value = evalValue(n, R); + return Array.isArray(value) ? value : [value]; } function aggNumbers(args: Node[], R: CellResolver): number[] { const out: number[] = []; @@ -359,12 +376,63 @@ function matchesCriteria(value: CellValue, criteria: CellValue): boolean { return looseEqual(value, criteria); } -function evalCall(n: { name: string; args: Node[] }, R: CellResolver): CellValue { +const MS_PER_DAY = 24 * 60 * 60 * 1000; + +function excelDateParts(serial: number): { day: number; month: number; year: number; weekday: string } { + const wholeDays = Math.trunc(serial); + if (!Number.isFinite(serial) || wholeDays < 0) throw new FormulaEvalError("#VALUE!"); + const weekdayIndex = ((wholeDays - 1) % 7 + 7) % 7; + const weekday = ["Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday"][weekdayIndex]; + if (wholeDays === 60) return { day: 29, month: 2, year: 1900, weekday }; + const epoch = wholeDays < 60 ? Date.UTC(1899, 11, 31) : Date.UTC(1899, 11, 30); + const date = new Date(epoch + wholeDays * MS_PER_DAY); + if (Number.isNaN(date.getTime())) throw new FormulaEvalError("#VALUE!"); + return { + day: date.getUTCDate(), + month: date.getUTCMonth() + 1, + year: date.getUTCFullYear(), + weekday, + }; +} + +function formatExcelDateSerial(serial: number, format: string): string { + const parts = excelDateParts(serial); + const monthNames = ["January", "February", "March", "April", "May", "June", "July", "August", "September", "October", "November", "December"]; + const replacements: Record = { + dddd: parts.weekday, + ddd: parts.weekday.slice(0, 3), + dd: String(parts.day).padStart(2, "0"), + d: String(parts.day), + mmmm: monthNames[parts.month - 1], + mmm: monthNames[parts.month - 1].slice(0, 3), + mm: String(parts.month).padStart(2, "0"), + m: String(parts.month), + yyyy: String(parts.year).padStart(4, "0"), + yy: String(parts.year % 100).padStart(2, "0"), + }; + const unsupported = format.replace(/dddd|ddd|dd|d|mmmm|mmm|mm|m|yyyy|yy/gi, ""); + if (/[a-z]/i.test(unsupported)) throw new FormulaEvalError("#UNSUPPORTED!"); + let matched = false; + const rendered = format.replace(/dddd|ddd|dd|d|mmmm|mmm|mm|m|yyyy|yy/gi, (token) => { + matched = true; + return replacements[token.toLowerCase()]; + }); + if (!matched) throw new FormulaEvalError("#UNSUPPORTED!"); + return rendered; +} + +function evalCall(n: { name: string; args: Node[] }, R: CellResolver): EvalValue { const fn = n.name; if (!SUPPORTED.has(fn)) throw new FormulaEvalError("#NAME?"); switch (fn) { case "SUM": return round12(aggNumbers(n.args, R).reduce((s, x) => s + x, 0)); case "AVERAGE": { const xs = aggNumbers(n.args, R); if (xs.length === 0) throw new FormulaEvalError("#DIV/0!"); return round12(xs.reduce((s, x) => s + x, 0) / xs.length); } + case "MEDIAN": { + const xs = aggNumbers(n.args, R).sort((a, b) => a - b); + if (xs.length === 0) throw new FormulaEvalError("#NUM!"); + const middle = Math.floor(xs.length / 2); + return xs.length % 2 === 1 ? xs[middle] : round12((xs[middle - 1] + xs[middle]) / 2); + } case "MIN": { const xs = aggNumbers(n.args, R); return xs.length ? Math.min(...xs) : 0; } case "MAX": { const xs = aggNumbers(n.args, R); return xs.length ? Math.max(...xs) : 0; } case "COUNT": return aggNumbers(n.args, R).length; @@ -408,15 +476,18 @@ function evalCall(n: { name: string; args: Node[] }, R: CellResolver): CellValue case "VLOOKUP": { const lookup = evalScalar(n.args[0], R); const table = rangeGrid(n.args[1], R); - const colIdx = Math.trunc(toNumber(evalScalar(n.args[2], R))); + const colArg = evalValue(n.args[2], R); + const returnsArray = Array.isArray(colArg); + const colIndices = (returnsArray ? colArg : [colArg]).map((value) => Math.trunc(toNumber(value))); const approx = n.args[3] ? truthy(evalScalar(n.args[3], R)) : true; - if (table.length === 0 || colIdx < 1 || colIdx > table[0].length) throw new FormulaEvalError("#REF!"); - if (!approx) { for (const row of table) if (looseEqual(row[0], lookup)) return row[colIdx - 1]; throw new FormulaEvalError("#N/A"); } + if (table.length === 0 || colIndices.some((colIdx) => colIdx < 1 || colIdx > table[0].length)) throw new FormulaEvalError("#REF!"); + const project = (row: CellValue[]): EvalValue => returnsArray ? colIndices.map((colIdx) => row[colIdx - 1]) : row[colIndices[0] - 1]; + if (!approx) { for (const row of table) if (looseEqual(row[0], lookup)) return project(row); throw new FormulaEvalError("#N/A"); } const ln = toNum(lookup); let best = -1; for (let i = 0; i < table.length; i++) { const cn = toNum(table[i][0]); if (cn !== null && ln !== null) { if (cn <= ln) best = i; else break; } } if (best < 0) throw new FormulaEvalError("#N/A"); - return table[best][colIdx - 1]; + return project(table[best]); } case "INDEX": { const grid = rangeGrid(n.args[0], R); @@ -453,6 +524,10 @@ function evalCall(n: { name: string; args: Node[] }, R: CellResolver): CellValue case "TRIM": return String(evalScalar(n.args[0], R) ?? "").trim().replace(/\s+/g, " "); case "UPPER": return String(evalScalar(n.args[0], R) ?? "").toUpperCase(); case "LOWER": return String(evalScalar(n.args[0], R) ?? "").toLowerCase(); + case "TEXT": { + if (n.args.length !== 2) throw new FormulaEvalError("#ERROR!"); + return formatExcelDateSerial(toNumber(evalScalar(n.args[0], R)), String(evalScalar(n.args[1], R) ?? "")); + } } throw new FormulaEvalError("#NAME?"); } diff --git a/src/nodeagent/core/runtime.ts b/src/nodeagent/core/runtime.ts index 2ab9499c..338d2f1b 100644 --- a/src/nodeagent/core/runtime.ts +++ b/src/nodeagent/core/runtime.ts @@ -780,11 +780,16 @@ export async function runAgent(opts: { // Goal-progress accounting for the two harness guards (read-loop breaker + done-without-writes // bounce). Counts WRITE-intent tool calls across the whole run; each guard fires at most once. const WRITE_TOOLS = new Set(["edit_cell", "create_draft", "update_wiki", "append_notebook_outline", "write_cell_result", "write_locked_cell", "write_locked_cell_result", "write_locked_cells", "write_locked_cell_results", "execute_verified_workbook_plan", "create_btb_deliverable_package"]); + const READ_TOOLS = new Set(["list_artifacts", "read_range", "search_sheet_context", "inspect_workbook", "fetch_source", "source_open", "source_open_literal"]); + const REQUIRED_GATE_READ_TOOLS = new Set(["inspect_workbook"]); let writeCalls = 0; let lockCalls = 0; let readNudged = false; let doneNudged = false; let requiredNoToolNudges = 0; + let preWriteSayCalls = 0; + let preWriteReadCalls = 0; + let readToolsWithheldNoted = false; let deterministicFailureKey: string | undefined; let deterministicFailureCount = 0; const managedWriteToolsAvailable = tools.some((tool) => tool.name.startsWith("write_locked_cell")); @@ -812,7 +817,10 @@ export async function runAgent(opts: { if (messages.length) { writeCalls = countToolResults(messages, WRITE_TOOLS, "success"); lockCalls = countToolCalls(messages, new Set(["propose_lock"])); + preWriteSayCalls = writeCalls === 0 ? countToolResults(messages, new Set(["say"]), "success") : 0; + preWriteReadCalls = writeCalls === 0 ? countToolResults(messages, READ_TOOLS, "success") : 0; readNudged = countUserNotes(messages, "HARNESS NOTE: every tool call so far has been a read.") > 0; + readToolsWithheldNoted = countUserNotes(messages, "HARNESS NOTE: the pre-write read budget is exhausted.") > 0; requiredNoToolNudges = requiredNoToolMissCount(messages); doneNudged = countUserNotes(messages, "HARNESS NOTE: the user asked for a write/fill/update") > 0 || countUserNotes(messages, "HARNESS NOTE: the user asked for a BTB deliverable package") > 0 @@ -870,8 +878,14 @@ export async function runAgent(opts: { const packageOnlyTools = goalRequiresPackage && btbPackageSuccesses === 0 && btbPackageNudges >= BTB_PACKAGE_ONLY_AFTER_NUDGES ? tools.filter((tool) => btbPackageTools.has(tool.name)) : tools; - const packageOnlyMode = packageOnlyTools.length > 0 && packageOnlyTools.every((tool) => btbPackageTools.has(tool.name)); - const offeredTools = packageOnlyTools.length ? packageOnlyTools : tools; + const withholdReads = goalRequiresWrite && writeCalls === 0 && readNudged && preWriteReadCalls >= 5; + const writeProgressTools = goalRequiresWrite && writeCalls === 0 + ? packageOnlyTools.filter((tool) => + !(preWriteSayCalls > 0 && tool.name === "say") + && !(withholdReads && ((READ_TOOLS.has(tool.name) && !REQUIRED_GATE_READ_TOOLS.has(tool.name)) || tool.name === "say"))) + : packageOnlyTools; + const packageOnlyMode = writeProgressTools.length > 0 && writeProgressTools.every((tool) => btbPackageTools.has(tool.name)); + const offeredTools = writeProgressTools.length ? writeProgressTools : tools; const requiresToolThisTurn = offeredTools.length > 0 && ( (goalRequiresPackage && btbPackageSuccesses === 0) || (goalRequiresWrite && writeCalls === 0) @@ -1043,6 +1057,7 @@ export async function runAgent(opts: { let toolCallsForTurn = out.toolCalls; let truncatedBtbToolCalls = 0; let failedBtbPackageThisTurn = false; + let preWriteSayThisTurn = false; if (goalRequiresPackage && btbPackageSuccesses === 0) { const packageCalls = out.toolCalls.filter((call) => btbPackageTools.has(call.tool)); if (packageCalls.length > 0 && packageCalls.length < out.toolCalls.length) { @@ -1078,6 +1093,7 @@ export async function runAgent(opts: { pendingToolCalls = toolCallsForTurn.slice(callIndex + 1); const result = await executeCall(call, step); if (WRITE_TOOLS.has(call.tool) && !toolResultFailed(result)) writeCalls++; + if (READ_TOOLS.has(call.tool) && !toolResultFailed(result) && writeCalls === 0) preWriteReadCalls += 1; if (btbPackageTools.has(call.tool)) { if (toolResultFailed(result)) { btbPackageFailures++; @@ -1120,6 +1136,10 @@ export async function runAgent(opts: { } if (call.tool === "say" && !toolResultFailed(result)) { finalText = sayTextFromArgs(call.args) ?? finalText; + if (goalRequiresWrite && writeCalls === 0) { + preWriteSayCalls += 1; + preWriteSayThisTurn = true; + } if (pendingToolCalls.length === 0 && !goalRequiresWrite && !goalRequiresPackage) { const doneResult = await finishDoneOrContinue(step, step + 1, { proposedStopReason: "done", @@ -1132,6 +1152,19 @@ export async function runAgent(opts: { } } pendingToolCalls = []; + if (preWriteSayThisTurn && writeCalls === 0) { + messages.push({ + role: "user", + content: `HARNESS NOTE: status/chat output does not satisfy this write-required task. The say tool is now withheld until artifact progress is made. ${finishWriteInstruction}`, + }); + } + if (goalRequiresWrite && writeCalls === 0 && readNudged && preWriteReadCalls >= 5 && !readToolsWithheldNoted) { + readToolsWithheldNoted = true; + messages.push({ + role: "user", + content: `HARNESS NOTE: the pre-write read budget is exhausted. Broad read/search tools are withheld for the next turn; use the workbook evidence already in context to preflight and perform the required write. ${finishWriteInstruction}`, + }); + } // Read-loop breaker: 3+ full turns of pure reads with no lock/write yet → ONE steering note, // appended AFTER this turn's tool results so the tool_use/tool_result pairing stays intact. diff --git a/src/nodeagent/skills/spreadsheet/workbookTaskIntelligence.ts b/src/nodeagent/skills/spreadsheet/workbookTaskIntelligence.ts index 7ac02267..9c7860a7 100644 --- a/src/nodeagent/skills/spreadsheet/workbookTaskIntelligence.ts +++ b/src/nodeagent/skills/spreadsheet/workbookTaskIntelligence.ts @@ -29,6 +29,7 @@ export type WorkbookInspectionFindingKind = | "named_year_target_band" | "semantic_formula_target" | "formula_range_anomaly" + | "lookup_bounds_missing" | "implicit_assignment_target"; export type WorkbookInspectionFinding = { @@ -50,6 +51,13 @@ export type WorkbookTargetBand = { reason: string; }; +export type WorkbookBlockedTarget = { + sheet: string; + address: string; + reason: string; + missingDependencies: string[]; +}; + export type WorkbookTaskInspection = { schema: 1; mutatingTask: boolean; @@ -57,6 +65,7 @@ export type WorkbookTaskInspection = { referencedSheets: string[]; explicitReferences: WorkbookTaskReference[]; targetCandidates: Array<{ sheet: string; address: string; reason: string }>; + blockedTargets: WorkbookBlockedTarget[]; targetBands: WorkbookTargetBand[]; dependencyCandidates: Array<{ sheet: string; address: string; reason: string }>; findings: WorkbookInspectionFinding[]; @@ -125,6 +134,7 @@ export type WorkbookPlanIssueKind = | "formula_self_reference" | "formula_semantic_mismatch" | "value_semantic_mismatch" + | "unsafe_lookup_bounds" | "malformed_formula" | "duplicate_target"; @@ -200,6 +210,11 @@ type WorkbookTemplateCompletion = { values?: WorkbookTaskInspection["valueSuggestions"]; }; +type WorkbookLookupPassFailContract = { + completions: WorkbookTemplateCompletion[]; + blockedTargets: WorkbookBlockedTarget[]; +}; + const A1_RE = /^\$?([A-Z]{1,3})\$?([1-9][0-9]*)$/i; const CELL_TOKEN_RE = /\$?[A-Z]{1,3}\$?[1-9][0-9]*/gi; const GENERIC_ELEMENT_RE = /\b[a-z][a-z0-9_]*__[a-z][a-z0-9_]*\b/gi; @@ -239,6 +254,7 @@ export function buildWorkbookSuggestedPlan( sheet: string, ): WorkbookSuggestedPlan { const targetSheet = sheet.trim().toLowerCase(); + const targetKeys = new Set(inspection.targetCandidates.map((target) => workbookCellKey(target.sheet, target.address))); const candidates = new Map(); const add = (operation: WorkbookSuggestedPlanOperation) => { const elementId = normalizeAddress(operation.elementId); @@ -263,6 +279,7 @@ export function buildWorkbookSuggestedPlan( } for (const suggestion of inspection.formulaRepairSuggestions) { if (suggestion.confidence !== "high" || suggestion.sheet.trim().toLowerCase() !== targetSheet) continue; + if (!targetKeys.has(workbookCellKey(suggestion.sheet, suggestion.cell))) continue; add({ elementId: suggestion.cell, formula: suggestion.formula }); } for (const suggestion of inspection.valueSuggestions) { @@ -347,6 +364,7 @@ export function inspectWorkbookTask(args: { const formulaFillSuggestions: WorkbookTaskInspection["formulaFillSuggestions"] = []; const formulaRepairSuggestions: WorkbookTaskInspection["formulaRepairSuggestions"] = []; const valueSuggestions: WorkbookTaskInspection["valueSuggestions"] = []; + const blockedTargets: WorkbookBlockedTarget[] = []; const targetCandidates = new Map(); const targetBands = new Map(); const dependencyCandidates = new Map(); @@ -472,53 +490,73 @@ export function inspectWorkbookTask(args: { } } + const weekdayTargetKeys = new Set(); if (/\b(?:weekday|day\s+name|mon\b|wed\b)\b/i.test(args.instruction)) { + const preserveWeekdayPeers = /\b(?:only\s+(?:the\s+)?(?:anchor|named|specified|incorrect|wrong)|preserve|leave)\b[^.!?]{0,80}\b(?:other|adjacent|peer|existing)\b/i.test(args.instruction); + const normalizeWeekdayBand = !preserveWeekdayPeers + && /\b(?:mon|tue|wed|thu|fri|sat|sun)(?:day)?\b/i.test(args.instruction); + const quotedWeekdayTargets = new Set(findings + .filter((finding) => finding.kind === "formula_text_match") + .map((finding) => workbookCellKey(finding.sheet, finding.address))); for (const cell of args.cells) { const band = weekdayFormulaFillBand(cell, cellsByKey); if (band.length < 3) continue; - for (const target of band) { + const cellKey = workbookCellKey(cell.sheet, cell.address); + if (preserveWeekdayPeers && quotedWeekdayTargets.size > 0 && !quotedWeekdayTargets.has(cellKey)) continue; + const sourceFormula = weekdayTextFormula(cell.formula!, requestedWeekdayTextToken(args.instruction, band)); + const anchorPosition = parseAddress(cell.address)!; + const operationBand = preserveWeekdayPeers ? [cell] : band; + const operations = operationBand.flatMap((target) => { + const targetPosition = parseAddress(target.address)!; + const formula = translateRelativeFormula(sourceFormula, targetPosition.row - anchorPosition.row, targetPosition.col - anchorPosition.col); + const shouldRepair = preserveWeekdayPeers || normalizeFormula(target.formula) !== undefined + ? !weekdayFormulasEquivalent(target.formula!, formula) + : isBlank(unwrapCellValue(target.value)) || normalizeWeekdayBand; + return shouldRepair ? [{ sheet: target.sheet, cell: target.address, formula }] : []; + }); + if (operations.length === 0) continue; + for (const operation of operations) weekdayTargetKeys.add(workbookCellKey(operation.sheet, operation.cell)); + const operationKeys = new Set(operations.map((operation) => workbookCellKey(operation.sheet, operation.cell))); + const operationCells = band.filter((target) => operationKeys.has(workbookCellKey(target.sheet, target.address))); + const preservedPeers = band.length - operationCells.length; + for (const target of operationCells) { addRank(target, 220, "formula_fill_band"); - addCandidate("target", target, `visible weekday labels form a contiguous formula-fill band anchored at ${cell.address}`); + addCandidate("target", target, `visible weekday examples justify a bounded formula repair anchored at ${cell.address}`); } addTargetBand( cell.sheet, - band.map((target) => target.address), + operationCells.map((target) => target.address), "formula_fill", "high", - `visible weekday labels form a contiguous formula-fill band anchored at ${cell.address}`, + `visible weekday examples justify a bounded formula repair anchored at ${cell.address}`, ); findings.push({ kind: "formula_fill_band", severity: "warning", sheet: cell.sheet, address: cell.address, - relatedAddresses: band.filter((target) => target.address !== cell.address).map((target) => target.address), - detail: `${cell.address} and ${band.length - 1} adjacent weekday-label cells sit directly above date inputs.`, - recommendedAction: `Repair the anchor formula and fill the same relative TEXT formula across ${band[0].address}:${band.at(-1)!.address}; verify every target.`, + relatedAddresses: operationCells.filter((target) => target.address !== cell.address).map((target) => target.address), + detail: `${cell.address} sits in a ${band.length}-cell weekday row above date inputs; ${preservedPeers} populated label example(s) remain unchanged.`, + recommendedAction: normalizeWeekdayBand + ? `Normalize ${operations.map((operation) => operation.cell).join(", ")} with relative three-letter TEXT formulas and verify the full visible weekday row.` + : `Repair only ${operations.map((operation) => operation.cell).join(", ")} with the relative TEXT formula and preserve populated scalar labels.`, }); - const sourceFormula = weekdayTextFormula(cell.formula!); - const anchorPosition = parseAddress(cell.address)!; formulaFillSuggestions.push({ sheet: cell.sheet, - range: `${band[0].address}:${band.at(-1)!.address}`, + range: `${operationCells[0].address}:${operationCells.at(-1)!.address}`, anchorAddress: cell.address, sourceFormula, - operations: band.map((target) => { - const targetPosition = parseAddress(target.address)!; - return { - sheet: target.sheet, - cell: target.address, - formula: translateRelativeFormula(sourceFormula, targetPosition.row - anchorPosition.row, targetPosition.col - anchorPosition.col), - }; - }), + operations, }); } } const formulaBandAnalysis = analyzeFormulaBands(args.cells, addRank); - findings.push(...formulaBandAnalysis.findings); + findings.push(...formulaBandAnalysis.findings.filter((finding) => + !weekdayTargetKeys.has(workbookCellKey(finding.sheet, finding.address)))); const requireFormulaAnomalyRepairs = genericFormulaAuditTask(args.instruction); for (const suggestion of formulaBandAnalysis.suggestions) { + if (weekdayTargetKeys.has(workbookCellKey(suggestion.sheet, suggestion.cell))) continue; formulaRepairSuggestions.push(suggestion); if (requireFormulaAnomalyRepairs) { addCandidate("target", cellsByKey.get(workbookCellKey(suggestion.sheet, suggestion.cell)) ?? { @@ -670,7 +708,30 @@ export function inspectWorkbookTask(args: { } } - const templateCompletions = inferWorkbookTemplateCompletions(args.instruction, args.cells); + const lookupPassFailContract = inferLookupRangePassFailContract(args.instruction, args.cells); + for (const blocked of lookupPassFailContract.blockedTargets) { + blockedTargets.push(blocked); + const target = cellsByKey.get(workbookCellKey(blocked.sheet, blocked.address)) ?? { + sheet: blocked.sheet, + address: blocked.address, + value: "", + }; + addRank(target, 245, "lookup_bounds_missing"); + addCandidate("target", target, blocked.reason); + findings.push({ + kind: "lookup_bounds_missing", + severity: "error", + sheet: blocked.sheet, + address: blocked.address, + relatedAddresses: blocked.missingDependencies, + detail: blocked.reason, + recommendedAction: "Do not write a Pass/Fail result until every selected key has visible numeric lower and upper bounds.", + }); + } + const templateCompletions = [ + ...lookupPassFailContract.completions, + ...inferWorkbookTemplateCompletions(args.instruction, args.cells), + ]; for (const completion of templateCompletions) { const formulaAddresses = completion.formulas.map((operation) => operation.cell); addTargetBand( @@ -801,6 +862,7 @@ export function inspectWorkbookTask(args: { referencedSheets, explicitReferences, targetCandidates: [...targetCandidates.values()], + blockedTargets, targetBands: [...targetBands.values()], dependencyCandidates: [...dependencyCandidates.values()], findings: boundedFindings, @@ -960,6 +1022,140 @@ function inferWorkbookTemplateCompletions( ]; } +function inferLookupRangePassFailContract( + instruction: string, + cells: WorkbookObservedCell[], +): WorkbookLookupPassFailContract { + const empty = (): WorkbookLookupPassFailContract => ({ completions: [], blockedTargets: [] }); + if (!/\bpass\b[^.!?]{0,80}\bfail\b|\bfail\b[^.!?]{0,80}\bpass\b/i.test(instruction)) return empty(); + if (!/\b(?:corresponding|lookup|dropdown|range\s+defined|minimum\s+and\s+maximum)\b/i.test(instruction)) return empty(); + const sheetNames = [...new Set(cells.map((cell) => cell.sheet))]; + const references = extractWorkbookTaskReferences(instruction, sheetNames); + const completions: WorkbookTemplateCompletion[] = []; + const blockedTargets: WorkbookBlockedTarget[] = []; + + for (const sheet of sheetNames) { + const sheetCells = cells.filter((cell) => cell.sheet === sheet); + const byAddress = new Map(sheetCells.map((cell) => [normalizeAddress(cell.address), cell])); + const targetReference = references + .filter((reference) => reference.role === "target" && (!reference.sheet || reference.sheet.toLowerCase() === sheet.toLowerCase())) + .map((reference) => ({ reference, start: parseAddress(reference.start), end: parseAddress(reference.end) })) + .find(({ start, end }) => { + if (!start || !end || start.col !== end.col || end.row - start.row < 1 || end.row - start.row > 100) return false; + return Array.from({ length: end.row - start.row + 1 }, (_, index) => start.row + index).every((row) => { + const cell = byAddress.get(addressFromPosition(row, start.col)); + return !!cell && (isBlank(unwrapCellValue(cell.value)) || !!cell.formula); + }); + }); + if (!targetReference?.start || !targetReference.end) continue; + const targetRows = Array.from( + { length: targetReference.end.row - targetReference.start.row + 1 }, + (_, index) => targetReference.start!.row + index, + ); + const dependencyBands = references + .filter((reference) => reference !== targetReference.reference + && (!reference.sheet || reference.sheet.toLowerCase() === sheet.toLowerCase())) + .map((reference) => ({ reference, start: parseAddress(reference.start), end: parseAddress(reference.end) })) + .filter(({ start, end }) => start && end && start.col === end.col && start.row === targetReference.start!.row && end.row === targetReference.end!.row); + const keyBand = dependencyBands.find(({ start }) => targetRows.every((row) => { + const value = unwrapCellValue(byAddress.get(addressFromPosition(row, start!.col))?.value); + return typeof value === "string" && value.trim().length > 0; + })); + const measureBand = dependencyBands.find(({ start }) => targetRows.every((row) => { + const value = unwrapCellValue(byAddress.get(addressFromPosition(row, start!.col))?.value); + return typeof value === "number" && Number.isFinite(value); + })); + if (!keyBand?.start || !measureBand?.start) continue; + const selectedKeys = new Set(targetRows.map((row) => String(unwrapCellValue(byAddress.get(addressFromPosition(row, keyBand.start!.col))?.value)).trim().toLowerCase())); + + const keyColumns = groupByMap(sheetCells.filter((cell) => { + const value = unwrapCellValue(cell.value); + return typeof value === "string" && /^[A-Z]$/i.test(value.trim()) && !!parseAddress(cell.address); + }), (cell) => parseAddress(cell.address)!.col); + const lookupBand = [...keyColumns.values()] + .map((columnCells) => [...columnCells].sort((left, right) => parseAddress(left.address)!.row - parseAddress(right.address)!.row)) + .map((columnCells) => longestContiguousKeyRun(columnCells)) + .filter((columnCells) => columnCells.length >= 2 && [...selectedKeys].every((key) => columnCells.some((cell) => displayValue(cell.value).trim().toLowerCase() === key))) + .sort((left, right) => right.length - left.length)[0]; + if (!lookupBand) continue; + const lookupStart = parseAddress(lookupBand[0].address)!; + const lookupEnd = parseAddress(lookupBand.at(-1)!.address)!; + + const boundReferences = references + .filter((reference) => reference !== targetReference.reference + && reference.start === reference.end + && (!reference.sheet || reference.sheet.toLowerCase() === sheet.toLowerCase())) + .map((reference) => parseAddress(reference.start)) + .filter((position): position is CellPosition => !!position && position.row === lookupStart.row && position.col > lookupStart.col) + .sort((left, right) => left.col - right.col); + const adjacentBounds = boundReferences.find((position, index) => + boundReferences[index + 1]?.col === position.col + 1); + if (!adjacentBounds) continue; + const upperBoundCol = adjacentBounds.col + 1; + const missingBounds = targetRows.flatMap((targetRow) => { + const targetAddress = addressFromPosition(targetRow, targetReference.start!.col); + const selectedKey = displayValue(byAddress.get(addressFromPosition(targetRow, keyBand.start!.col))?.value).trim(); + const lookupKey = lookupBand.find((cell) => displayValue(cell.value).trim().toLowerCase() === selectedKey.toLowerCase()); + if (!lookupKey) { + return [{ + sheet, + address: targetAddress, + reason: `${sheet}!${targetAddress} cannot be safely classified because selected key ${JSON.stringify(selectedKey)} has no visible lookup row.`, + missingDependencies: [addressFromPosition(lookupStart.row, lookupStart.col)], + } satisfies WorkbookBlockedTarget]; + } + const lookupRow = parseAddress(lookupKey.address)!.row; + const lowerAddress = addressFromPosition(lookupRow, adjacentBounds.col); + const upperAddress = addressFromPosition(lookupRow, upperBoundCol); + const lower = unwrapCellValue(byAddress.get(lowerAddress)?.value); + const upper = unwrapCellValue(byAddress.get(upperAddress)?.value); + const lowerValid = typeof lower === "number" && Number.isFinite(lower); + const upperValid = typeof upper === "number" && Number.isFinite(upper); + if (lowerValid && upperValid && lower <= upper) return []; + const missingDependencies = [ + ...(!lowerValid ? [lowerAddress] : []), + ...(!upperValid ? [upperAddress] : []), + ...(lowerValid && upperValid && lower > upper ? [lowerAddress, upperAddress] : []), + ]; + return [{ + sheet, + address: targetAddress, + reason: `${sheet}!${targetAddress} cannot be safely classified because selected key ${JSON.stringify(selectedKey)} has missing, non-numeric, or reversed bounds at ${lowerAddress}:${upperAddress}.`, + missingDependencies, + } satisfies WorkbookBlockedTarget]; + }); + if (missingBounds.length > 0) { + blockedTargets.push(...missingBounds); + continue; + } + const tableRange = `$${columnNumberToName(lookupStart.col)}$${lookupStart.row}:$${columnNumberToName(upperBoundCol)}$${lookupEnd.row}`; + const lookupIndices = `{${adjacentBounds.col - lookupStart.col + 1},${upperBoundCol - lookupStart.col + 1}}`; + const formulas = targetRows.map((row) => ({ + sheet, + cell: addressFromPosition(row, targetReference.start!.col), + formula: `IF(MEDIAN(${addressFromPosition(row, measureBand.start!.col)},VLOOKUP(${addressFromPosition(row, keyBand.start!.col)},${tableRange},${lookupIndices},0))=${addressFromPosition(row, measureBand.start!.col)},"Pass","Fail")`, + })); + completions.push({ + sheet, + reason: "Visible selector, measurement, lookup keys, and adjacent lower/upper bound columns define a bounded pass/fail lookup contract", + formulas, + }); + } + return { completions, blockedTargets }; +} + +function longestContiguousKeyRun(cells: WorkbookObservedCell[]): WorkbookObservedCell[] { + let best: WorkbookObservedCell[] = []; + let current: WorkbookObservedCell[] = []; + for (const cell of cells) { + const row = parseAddress(cell.address)!.row; + const previousRow = current.length > 0 ? parseAddress(current.at(-1)!.address)!.row : undefined; + current = previousRow !== undefined && row === previousRow + 1 ? [...current, cell] : [cell]; + if (current.length > best.length) best = current; + } + return best; +} + function inferDebtWaterfallTemplateCompletion( instruction: string, cells: WorkbookObservedCell[], @@ -1751,9 +1947,19 @@ function repeatedFormulaFillSuggestion( }; } -function weekdayTextFormula(formula: string): string { +function requestedWeekdayTextToken(instruction: string, band: WorkbookObservedCell[]): "DDD" | "DDDD" { + if (/\b(?:full|long)\s+(?:weekday|day)\s+names?\b|\b(?:monday|tuesday|wednesday|thursday|friday|saturday|sunday)\b/i.test(instruction)) { + return "DDDD"; + } + if (/\b(?:abbreviat(?:e|ed|ion)|short)\b|\b(?:mon|tue|wed|thu|fri|sat|sun)\b/i.test(instruction)) return "DDD"; + const visibleTokens = band.flatMap((cell) => normalizeFormula(cell.formula)?.match(/TEXT\([^,]+,"(D{3,4})"\)$/i)?.[1].toUpperCase() ?? []); + return visibleTokens.includes("DDDD") ? "DDDD" : "DDD"; +} + +function weekdayTextFormula(formula: string, token: "DDD" | "DDDD"): string { const normalized = normalizeFormula(formula) ?? formula.trim().replace(/^=/, ""); - return normalized.replace(/(TEXT\([^,]+,")D{1,2}("\))$/i, "$1DDD$2"); + return normalized.replace(/(TEXT\([^,]+,")(D{1,4})("\))$/i, (match, prefix: string, currentToken: string, suffix: string) => + currentToken.length === token.length ? match : `${prefix}${token}${suffix}`); } function translateRelativeFormula(formula: string, rowDelta: number, colDelta: number): string { @@ -1838,16 +2044,14 @@ export function verifyWorkbookPlan(args: { const coveredTargets = new Set(); const seenTargets = new Map(); const targetKeys = new Set(args.inspection.targetCandidates.map((target) => workbookCellKey(target.sheet, target.address))); + const blockedTargetByKey = new Map((args.inspection.blockedTargets ?? []) + .map((target) => [workbookCellKey(target.sheet, target.address), target] as const)); const quotedFormulaTargets = new Set(args.inspection.findings .filter((finding) => finding.kind === "formula_text_match") .map((finding) => workbookCellKey(finding.sheet, finding.address))); const targetBandTargets = new Set((args.inspection.targetBands ?? []) .flatMap((band) => band.addresses.map((address) => workbookCellKey(band.sheet, address)))); const formulaBandTargets = new Set([ - ...args.inspection.findings - .filter((finding) => finding.kind === "formula_fill_band") - .flatMap((finding) => [finding.address, ...(finding.relatedAddresses ?? [])] - .map((address) => workbookCellKey(finding.sheet, address))), ...args.inspection.formulaFillSuggestions .flatMap((suggestion) => suggestion.operations.map((operation) => workbookCellKey(operation.sheet, operation.cell))), ]); @@ -1943,6 +2147,18 @@ export function verifyWorkbookPlan(args: { } const key = workbookCellKey(sheet, address); if (requiredTargetKeys.has(key)) coveredTargets.add(key); + const blockedTarget = blockedTargetByKey.get(key); + if (blockedTarget) { + issues.push({ + kind: "unsafe_lookup_bounds", + severity: "error", + operationIndex, + sheet, + address, + detail: blockedTarget.reason, + repair: `Populate and verify the visible bound dependencies first: ${blockedTarget.missingDependencies.join(", ")}.`, + }); + } const priorIndex = seenTargets.get(key); if (priorIndex !== undefined) { issues.push({ @@ -2115,7 +2331,7 @@ function weekdayFormulaFillBand( const source = cellsByKey.get(workbookCellKey(anchor.sheet, addressFromPosition(dependency.row, col))); if (!target || !source || !dateLikeWorkbookValue(source.value)) return undefined; const label = displayValue(target.value).trim(); - if (!target.formula && !/^(?:M|MO|MON|MONDAY|TU|TUE|TUESDAY|W|WED|WEDNESDAY|TH|THU|THURSDAY|F|FRI|FRIDAY|S|SA|SAT|SATURDAY|SU|SUN|SUNDAY)$/i.test(label)) return undefined; + if (!target.formula && label && !/^(?:M|MO|MON|MONDAY|TU|TUE|TUESDAY|W|WED|WEDNESDAY|TH|THU|THURSDAY|F|FRI|FRIDAY|S|SA|SAT|SATURDAY|SU|SUN|SUNDAY)$/i.test(label)) return undefined; return target; }; let start = position.col; @@ -2127,10 +2343,14 @@ function weekdayFormulaFillBand( const cell = qualifies(col); if (cell) band.push(cell); } - if (band.filter((cell) => cell.address !== anchor.address && !cell.formula).length < 2) return []; + if (band.length < 3) return []; return band; } +function weekdayFormulasEquivalent(left: string, right: string): boolean { + return normalizeFormula(left)?.toUpperCase() === normalizeFormula(right)?.toUpperCase(); +} + function dateLikeWorkbookValue(value: unknown): boolean { const raw = unwrapCellValue(value); if (raw instanceof Date && Number.isFinite(raw.getTime())) return true; @@ -2200,8 +2420,20 @@ export function checksForWorkbookOperations(operations: WorkbookPlanOperation[]) } function referenceRole(instruction: string, index: number, length: number): WorkbookReferenceRole { - const before = instruction.slice(Math.max(0, index - 90), index); - const after = instruction.slice(index + length, Math.min(instruction.length, index + length + 90)); + const before = instruction.slice(Math.max(0, index - 180), index); + const after = instruction.slice(index + length, Math.min(instruction.length, index + length + 180)); + const clauseBefore = before.slice(Math.max(before.lastIndexOf("."), before.lastIndexOf(";"), before.lastIndexOf("\n")) + 1); + const clauseAfter = after.split(/[.;\n]/, 1)[0] ?? after; + if (/=[A-Z][A-Z0-9_.]*\([^;!?\n]{0,180}$/i.test(before)) return "dependency"; + if (/\b[A-Z][A-Z0-9_.]*\([^()]{0,180}$/i.test(before)) return "dependency"; + if (/=[^=.!?;\n]{0,180}$/i.test(clauseBefore)) return "dependency"; + if (/\b(?:users?|they)\s+(?:input|enter|select|provide)\b[^.!?;\n]{0,120}\b(?:in|into)\s+(?:cells?|range|columns?)\s*['"]?$/i.test(clauseBefore)) return "dependency"; + if (/\b(?:range|limits?|bounds?)\s+(?:is|are\s+)?defined\s+in\s+(?:cells?|range|columns?)\b[^.!?;\n]{0,60}$/i.test(clauseBefore)) return "dependency"; + if (/\b(?:dropdown|selector)\b[^.!?;\n]{0,80}\b(?:populates?|fills?)\s*$/i.test(clauseBefore)) return "dependency"; + if (/\bcorresponding\s+(?:cell|value)\b[^.!?;\n]{0,80}$/i.test(clauseBefore)) return "dependency"; + if (/^\s*[,)]?\s*(?:i|we|it|which)\s+(?:have|has|uses?|contains?)\b/i.test(clauseAfter)) return "dependency"; + if (/\bi\s+(?:need|want)\b[^.!?;\n]{0,100}\b(?:cells?|range|column)\b[^.!?;\n]{0,30}$/i.test(clauseBefore) + && /^\s*\)?\s*to\s+(?:check|show|display|return|calculate|verify|populate|contain)/i.test(clauseAfter)) return "target"; if (/\b(?:set|write|fill|populate|place|put|output|configure|correct|fix|repair|change|replace|calculate)(?:\s+(?:cells?|range|column))?\s*$/i.test(before)) return "target"; if (/\b(?:from|using|based\s+on|criteria\s+in|source|input)(?:\s+(?:cells?|range|column))?\s*$/i.test(before)) return "dependency"; const immediate = `${before.slice(-55)} ${after.slice(0, 55)}`; diff --git a/tests/agentRuntime.test.ts b/tests/agentRuntime.test.ts index 1f3da531..be5979dd 100644 --- a/tests/agentRuntime.test.ts +++ b/tests/agentRuntime.test.ts @@ -203,6 +203,32 @@ describe("agent runtime — collaboration under concurrency", () => { expect(r.trace.filter((t) => t.tool === "say")).toHaveLength(1); }); + it("withholds repeated say calls until a write-required goal makes artifact progress", async () => { + const { rt } = setup(); + const offeredToolNames: string[][] = []; + let turn = 0; + const model: AgentModel = { + name: "pre-write-say-loop", + async next({ tools }) { + offeredToolNames.push(tools.map((tool) => tool.name)); + turn += 1; + if (turn === 1) return { + toolCalls: [{ id: "say-before-write", tool: "say", args: { text: "I am still working." } }], + done: false, + }; + return { text: "No write performed.", toolCalls: [], done: true }; + }, + }; + + const result = await runAgent({ rt, goal: "fill the sheet cells", model, tools: ROOM_TOOLS, maxSteps: 2 }); + + expect(offeredToolNames[0]).toContain("say"); + expect(offeredToolNames[1]).not.toContain("say"); + expect(result.trace.filter((event) => event.tool === "say")).toHaveLength(1); + expect(result.messages.some((message) => message.content?.includes("say tool is now withheld"))).toBe(true); + expect(result.stopReason).toBe("step_budget"); + }); + it("does not steer read-only report answers into the BTB package tool", async () => { const { rt } = setup(); let packageCalls = 0; @@ -343,6 +369,72 @@ describe("agent runtime — collaboration under concurrency", () => { expect(result.trace.find((event) => event.tool === "edit_cell")?.result).toMatchObject({ ok: false }); }); + it("withholds broad reads after the bounded pre-write evidence budget is exhausted", async () => { + const { rt } = setup(); + const offeredToolNames: string[][] = []; + const model: AgentModel = { + name: "bounded-read-loop", + async next({ tools }) { + const names = tools.map((tool) => tool.name); + offeredToolNames.push(names); + if (names.includes("read_range")) { + return { toolCalls: [{ id: `read-${offeredToolNames.length}`, tool: "read_range", args: { elementIds: ["r_rev__variance"] } }], done: false }; + } + return { text: "No write performed.", toolCalls: [], done: true }; + }, + }; + + const result = await runAgent({ rt, goal: "complete the financial model", model, tools: ROOM_TOOLS, maxSteps: 6 }); + + expect(offeredToolNames).toHaveLength(6); + expect(offeredToolNames[4]).toContain("read_range"); + expect(offeredToolNames[5]).not.toContain("read_range"); + expect(offeredToolNames[5]).not.toContain("say"); + expect(result.trace.filter((event) => event.tool === "read_range")).toHaveLength(5); + expect(result.messages.some((message) => message.content?.includes("pre-write read budget is exhausted"))).toBe(true); + expect(result.stopReason).toBe("step_budget"); + }); + + it("keeps mandatory workbook inspection available after exploratory reads are withheld", async () => { + const { rt } = setup(); + const offeredToolNames: string[][] = []; + const inspectTool: AgentTool = { + name: "inspect_workbook", + description: "Required workbook workflow inspection gate.", + schema: z.object({}), + execute: async () => ({ ok: true, inspection: { schema: 1 } }), + }; + const model: AgentModel = { + name: "bounded-read-inspection-gate", + async next({ tools }) { + const names = tools.map((tool) => tool.name); + offeredToolNames.push(names); + if (offeredToolNames.length <= 5) { + return { toolCalls: [{ id: `read-${offeredToolNames.length}`, tool: "read_range", args: { elementIds: ["r_rev__variance"] } }], done: false }; + } + return { toolCalls: [{ id: "inspect-required", tool: "inspect_workbook", args: {} }], done: false }; + }, + }; + + const result = await runAgent({ + rt, + goal: "complete the financial model", + model, + tools: [ + ...ROOM_TOOLS.filter((tool) => tool.name !== "inspect_workbook"), + inspectTool, + ], + maxSteps: 6, + }); + + expect(offeredToolNames[5]).not.toContain("read_range"); + expect(offeredToolNames[5]).toContain("inspect_workbook"); + expect(result.trace.find((event) => event.tool === "inspect_workbook")).toMatchObject({ + tool: "inspect_workbook", + result: { ok: true }, + }); + }); + it("steers read-looping BTB runs toward the deliverable package tool", async () => { const { rt } = setup(); let sawPackageNudge = false; diff --git a/tests/formulaEngine.test.ts b/tests/formulaEngine.test.ts index b4706648..04fc2d3b 100644 --- a/tests/formulaEngine.test.ts +++ b/tests/formulaEngine.test.ts @@ -126,6 +126,33 @@ describe("Functions and operators", () => { test("preserves Excel-scale forecast precision", () => { expect(val(makeSheet({ X: "=5620-1251.44166666667" }).compute("X"))).toBe(4368.55833333333); }); + test("TEXT formats Excel date serials with day and weekday tokens", () => { + const s = makeSheet({ + A1: 45292, + X1: '=TEXT(A1,"DDD")', + X2: '=TEXT(A1,"dddd")', + X3: '=TEXT(A1,"DD")', + X4: '=TEXT(A1,"mmmm d, yyyy")', + X5: '=TEXT(1,"ddd")', + X6: '=TEXT(60,"dddd")', + X7: '=TEXT(0,"ddd")', + }); + expect(val(s.compute("X1"))).toBe("Mon"); + expect(val(s.compute("X2"))).toBe("Monday"); + expect(val(s.compute("X3"))).toBe("01"); + expect(val(s.compute("X4"))).toBe("January 1, 2024"); + expect(val(s.compute("X5"))).toBe("Sun"); + expect(val(s.compute("X6"))).toBe("Wednesday"); + expect(val(s.compute("X7"))).toBe("Sat"); + }); + test("TEXT distinguishes valid but unsupported number formats from invalid formulas", () => { + expect(val(makeSheet({ X: '=TEXT(2,"0.00")' }).compute("X"))).toBe("ERR:#UNSUPPORTED!"); + }); + test("MEDIAN handles ranges and one-dimensional array constants", () => { + const s = makeSheet({ A1: 10, A2: 2, A3: 8, A4: "n/a", X1: "=MEDIAN(A1:A4)", X2: "=MEDIAN({9,10})" }); + expect(val(s.compute("X1"))).toBe(8); + expect(val(s.compute("X2"))).toBe(9.5); + }); }); describe("Error handling never crashes, never silently lies", () => { @@ -165,6 +192,16 @@ describe("Lookup + criteria functions (the finance power tools)", () => { test("AVERAGEIF — avg ARR for tier A", () => expect(val(makeSheet({ ...book, X: '=AVERAGEIF(B1:B4,"A",C1:C4)' }).compute("X"))).toBe(150)); test("VLOOKUP exact — ARR for Cive", () => expect(val(makeSheet({ ...book, X: '=VLOOKUP("Cive",A1:C4,3,FALSE)' }).compute("X"))).toBe(200)); test("VLOOKUP miss -> #N/A", () => expect(val(makeSheet({ ...book, X: '=VLOOKUP("Zzz",A1:C4,3,FALSE)' }).compute("X"))).toBe("ERR:#N/A")); + test("MEDIAN consumes VLOOKUP results for an array of column indices", () => { + const s = makeSheet({ + A3: "K1", I3: 10, J3: 20, + B15: "K1", H15: 15, + X1: '=MEDIAN(H15,VLOOKUP(B15,$A$3:$J$9,{9,10},FALSE))', + X2: '=IF(MEDIAN(H15,VLOOKUP(B15,$A$3:$J$9,{9,10},0))=H15,"Pass","Fail")', + }); + expect(val(s.compute("X1"))).toBe(15); + expect(val(s.compute("X2"))).toBe("Pass"); + }); test("INDEX/MATCH — ARR where name=Bolt", () => expect(val(makeSheet({ ...book, X: '=INDEX(C1:C4,MATCH("Bolt",A1:A4,0))' }).compute("X"))).toBe(50)); test("IFERROR wraps #DIV/0! with a fallback", () => expect(val(makeSheet({ X: '=IFERROR(1/0,"n/a")' }).compute("X"))).toBe("n/a")); test("IFERROR passes a good value through", () => expect(val(makeSheet({ X: "=IFERROR(2+2,0)" }).compute("X"))).toBe(4)); @@ -187,6 +224,6 @@ describe("Lookup + criteria functions (the finance power tools)", () => { }); test("unknown function still -> #NAME?", () => expect(val(makeSheet({ X: "=FOO(1)" }).compute("X"))).toBe("ERR:#NAME?")); test("SUPPORTED set exposes the new functions", () => { - for (const fn of ["SUMIF", "COUNTIF", "AVERAGEIF", "VLOOKUP", "INDEX", "MATCH", "IFERROR", "MOD", "POWER", "LEN"]) expect(SUPPORTED_FORMULA_FUNCTIONS).toContain(fn); + for (const fn of ["SUMIF", "COUNTIF", "AVERAGEIF", "VLOOKUP", "INDEX", "MATCH", "IFERROR", "MOD", "POWER", "LEN", "MEDIAN", "TEXT"]) expect(SUPPORTED_FORMULA_FUNCTIONS).toContain(fn); }); }); diff --git a/tests/spreadsheetBenchNodeAgentBridge.test.ts b/tests/spreadsheetBenchNodeAgentBridge.test.ts index 528db75f..ef967de6 100644 --- a/tests/spreadsheetBenchNodeAgentBridge.test.ts +++ b/tests/spreadsheetBenchNodeAgentBridge.test.ts @@ -108,6 +108,138 @@ describe("SpreadsheetBench canonical NodeAgent bridge", () => { ].includes(name)))).toBe(true); }); + it("adapts an invalid model-supplied inspection artifact to a visible worksheet", async () => { + const root = tempRoot(); + const instruction = "Inspect the workbook and report what remains."; + const task = await stagedTask(root, instruction); + const receipt = await runSpreadsheetBenchNodeAgentBridge({ + agentManifestPath: task.agentManifest, + candidateWorkbookPath: join(root, "output", "adapted-artifact.xlsx"), + model: invalidArtifactInspectionModel(), + traceId: "trace_sbench_adapted_artifact", + maxSteps: 4, + now: () => 1_200, + }); + + expect(receipt.frame.runtimeError).toBeUndefined(); + expect(receipt.stages.inspect.status).toBe("completed"); + expect(receipt.frame.agentResult.trace[0]).toMatchObject({ + tool: "inspect_workbook", + args: { artifactId: "Financial_Model/01_02" }, + result: { ok: true, artifactId: "Model" }, + }); + }); + + it("keeps a syntactically verified write open when deterministic recalculation is unsupported", async () => { + const root = tempRoot(); + const instruction = 'Replace Model!B1 with formula TEXT(A1,"0.00"), then verify the changed cell.'; + const task = await stagedTask(root, instruction); + const receipt = await runSpreadsheetBenchNodeAgentBridge({ + agentManifestPath: task.agentManifest, + candidateWorkbookPath: join(root, "output", "unresolved.xlsx"), + model: unresolvedRecalculationModel(instruction), + traceId: "trace_sbench_unresolved_recalculation", + maxSteps: 8, + now: () => 1_250, + }); + + expect(receipt.stages.verify.status).toBe("completed"); + expect(receipt.recalculation).toMatchObject({ + attemptedFormulaCount: 1, + refreshedFormulaCount: 0, + unresolvedFormulaCount: 1, + unresolved: [expect.objectContaining({ sheet: "Model", address: "B1", error: "#UNSUPPORTED!" })], + }); + expect(receipt.outcome).toMatchObject({ + status: "needs_repair", + changedCellCount: 1, + finalVerificationStatus: "needs_repair", + }); + expect(receipt.trace.final.status).toBe("needs_review"); + expect(receipt.trace.evidence.find((evidence) => evidence.label === "SpreadsheetBench formula recalculation")?.status).toBe("needs_review"); + }); + + it("never completes an unresolved formula write even when task wording is classified as nonmutating", async () => { + const root = tempRoot(); + const instruction = 'Model!B1 should contain formula TEXT(A1,"0.00").'; + const task = await stagedTask(root, instruction); + const receipt = await runSpreadsheetBenchNodeAgentBridge({ + agentManifestPath: task.agentManifest, + candidateWorkbookPath: join(root, "output", "unresolved-nonmutating.xlsx"), + model: unresolvedRecalculationModel(instruction), + traceId: "trace_sbench_unresolved_nonmutating", + maxSteps: 8, + now: () => 1_275, + }); + + expect(receipt.outcome).toMatchObject({ + status: "needs_repair", + mutatingTask: false, + changedCellCount: 1, + finalVerificationStatus: "needs_repair", + }); + expect(receipt.recalculation.unresolvedFormulaCount).toBe(1); + }); + + it("requires post-write verification after an observed write even when task wording is classified as nonmutating", async () => { + const root = tempRoot(); + const instruction = "Model!B1 should contain formula A1*2."; + const task = await stagedTask(root, instruction); + const receipt = await runSpreadsheetBenchNodeAgentBridge({ + agentManifestPath: task.agentManifest, + candidateWorkbookPath: join(root, "output", "missing-post-write-verification.xlsx"), + model: noPostWriteVerificationModel(instruction), + traceId: "trace_sbench_missing_post_write_verification", + maxSteps: 6, + now: () => 1_300, + }); + + expect(receipt.outcome).toMatchObject({ + status: "needs_repair", + mutatingTask: false, + changedCellCount: 1, + finalVerificationStatus: "missing", + }); + expect(receipt.stages.write.status).toBe("completed"); + expect(receipt.stages.verify.status).toBe("skipped"); + expect(receipt.trace.final.status).toBe("needs_review"); + }); + + it("blocks model-supplied pass/fail writes when a selected lookup key has missing bounds", async () => { + const root = tempRoot(); + const task = await stagedMissingLookupBoundsTask(root); + const candidate = join(root, "output", "missing-lookup-bounds.xlsx"); + const receipt = await runSpreadsheetBenchNodeAgentBridge({ + agentManifestPath: task.agentManifest, + candidateWorkbookPath: candidate, + model: missingLookupBoundsModel(task.instruction), + traceId: "trace_sbench_missing_lookup_bounds", + maxSteps: 6, + now: () => 1_350, + }); + + expect(receipt.outcome).toMatchObject({ + status: "needs_repair", + changedCellCount: 0, + finalVerificationStatus: "missing", + }); + expect(receipt.stages.preflight.status).toBe("needs_repair"); + const preflight = receipt.frame.agentResult.trace.find((event) => + event.tool === "verify_workbook" && (event.args as Record).afterWrite === false); + expect(preflight?.result).toMatchObject({ + status: "needs_repair", + plan: { + issues: expect.arrayContaining([expect.objectContaining({ kind: "unsafe_lookup_bounds", address: "J16" })]), + }, + }); + const write = receipt.frame.agentResult.trace.find((event) => event.tool === "write_locked_cells"); + expect(write?.result).toMatchObject({ ok: false, error: "workbook_stage_guard" }); + + const emitted = new ExcelJS.Workbook(); + await emitted.xlsx.readFile(candidate); + expect(emitted.getWorksheet("Sheet1")?.getCell("J16").value).toBeNull(); + }); + it("executes a large inferred plan through one compact managed action while preserving phase receipts", async () => { const root = tempRoot(); const task = await stagedDebtWaterfallTask(root); @@ -248,7 +380,8 @@ describe("SpreadsheetBench canonical NodeAgent bridge", () => { now: () => 4_000, }); - expect(receipt.outcome).toMatchObject({ status: "completed", changedCellCount: 3, finalVerificationStatus: "passed" }); + expect(receipt.outcome).toMatchObject({ status: "needs_repair", changedCellCount: 3, finalVerificationStatus: "needs_repair" }); + expect(receipt.recalculation.unresolvedFormulaCount).toBeGreaterThan(0); expect(receipt.frame.agentResult.stopReason).toBe("step_budget"); const inspection = receipt.frame.agentResult.trace.find((event) => event.tool === "inspect_workbook")?.result as Record; expect(inspection).toMatchObject({ @@ -296,7 +429,8 @@ describe("SpreadsheetBench canonical NodeAgent bridge", () => { now: () => 4_250, }); - expect(receipt.outcome).toMatchObject({ status: "completed", changedCellCount: 3, finalVerificationStatus: "passed" }); + expect(receipt.outcome).toMatchObject({ status: "needs_repair", changedCellCount: 3, finalVerificationStatus: "needs_repair" }); + expect(receipt.recalculation.unresolvedFormulaCount).toBeGreaterThan(0); const postWrites = receipt.frame.agentResult.trace.filter((event) => event.tool === "verify_workbook" && (event.args as Record).afterWrite === true); expect(postWrites).toHaveLength(2); @@ -310,7 +444,7 @@ describe("SpreadsheetBench canonical NodeAgent bridge", () => { expect(postWrites[1].result).toMatchObject({ status: "passed", workflowComplete: true }); }); - it("preserves a completed deterministic proof when the provider fails after verification", async () => { + it("does not preserve completion when the provider fails and recalculation remains unresolved", async () => { const root = tempRoot(); const task = await stagedWorkbookWideAverageTask(root); const receipt = await runSpreadsheetBenchNodeAgentBridge({ @@ -324,7 +458,8 @@ describe("SpreadsheetBench canonical NodeAgent bridge", () => { expect(receipt.frame.runtimeError).toContain("provider failed after deterministic proof"); expect(receipt.frame.agentResult.stopReason).toBe("error"); - expect(receipt.outcome).toMatchObject({ status: "completed", changedCellCount: 3, finalVerificationStatus: "passed" }); + expect(receipt.outcome).toMatchObject({ status: "failed", changedCellCount: 3, finalVerificationStatus: "needs_repair" }); + expect(receipt.recalculation.unresolvedFormulaCount).toBeGreaterThan(0); }); it("rejects an input path that escapes the staged agent directory before invoking the model", async () => { @@ -463,6 +598,70 @@ function repairingWorkbookModel(capture: { }; } +function unresolvedRecalculationModel(instruction: string): AgentModel { + let callIndex = 0; + const operation = { elementId: "B1", formula: 'TEXT(A1,"0.00")' }; + return { + name: "scripted-unresolved-recalculation", + async next(): Promise { + const id = `unresolved-recalculation-${++callIndex}`; + if (callIndex === 1) return step(id, "inspect_workbook", { instruction, artifactId: "Model", maxCells: 40 }); + if (callIndex === 2) return step(id, "verify_workbook", { instruction, artifactId: "Model", afterWrite: false, operations: [operation] }); + if (callIndex === 3) return step(id, "write_locked_cells", { artifactId: "Model", reason: "apply verified formula", ops: [operation] }); + if (callIndex === 4) return step(id, "verify_workbook", { instruction, artifactId: "Model", afterWrite: true, operations: [operation] }); + return { text: "The formula write passed structural verification.", toolCalls: [], done: true }; + }, + }; +} + +function noPostWriteVerificationModel(instruction: string): AgentModel { + let callIndex = 0; + const operation = { elementId: "B1", formula: "A1*2" }; + return { + name: "scripted-no-post-write-verification", + async next(): Promise { + const id = `no-post-write-verification-${++callIndex}`; + if (callIndex === 1) return step(id, "inspect_workbook", { instruction, artifactId: "Model", maxCells: 40 }); + if (callIndex === 2) return step(id, "verify_workbook", { instruction, artifactId: "Model", afterWrite: false, operations: [operation] }); + if (callIndex === 3) return step(id, "write_locked_cells", { artifactId: "Model", reason: "apply verified formula", ops: [operation] }); + return { text: "The requested formula was written.", toolCalls: [], done: true }; + }, + }; +} + +function missingLookupBoundsModel(instruction: string): AgentModel { + let callIndex = 0; + const operations = [ + { elementId: "J15", formula: 'IF(MEDIAN(H15,VLOOKUP(B15,$A$3:$J$4,{9,10},0))=H15,"Pass","Fail")' }, + { elementId: "J16", formula: 'IF(MEDIAN(H16,VLOOKUP(B16,$A$3:$J$4,{9,10},0))=H16,"Pass","Fail")' }, + ]; + return { + name: "scripted-missing-lookup-bounds", + async next(): Promise { + const id = `missing-lookup-bounds-${++callIndex}`; + if (callIndex === 1) return step(id, "inspect_workbook", { instruction, artifactId: "Sheet1", maxCells: 80 }); + if (callIndex === 2) return step(id, "verify_workbook", { instruction, artifactId: "Sheet1", afterWrite: false, operations }); + if (callIndex === 3) return step(id, "write_locked_cells", { artifactId: "Sheet1", reason: "attempt pass/fail formulas", ops: operations }); + return { text: "The unsafe plan was not applied.", toolCalls: [], done: true }; + }, + }; +} + +function invalidArtifactInspectionModel(): AgentModel { + let callIndex = 0; + return { + name: "scripted-invalid-artifact-inspection", + async next(): Promise { + callIndex += 1; + if (callIndex === 1) return step("invalid-artifact-inspection-1", "inspect_workbook", { + instruction: "Inspect the workbook and report what remains.", + artifactId: "Financial_Model/01_02", + }); + return { text: "Inspection complete.", toolCalls: [], done: true }; + }, + }; +} + function compactWorkbookPlanModel(): AgentModel { let callIndex = 0; return { @@ -660,6 +859,37 @@ async function stagedTask( return { agentManifest: join(agentDir, "task.json") }; } +async function stagedMissingLookupBoundsTask( + root: string, +): Promise<{ agentManifest: string; instruction: string }> { + const instruction = "Users select A-B in B15:B16 and enter readings in H15:H16. Fill J15:J16 with Pass or Fail using the corresponding lookup range defined by I3 and J3. The dropdown populates B3:B4."; + const taskDir = join(root, "tasks", "lookup-bounds"); + const agentDir = join(taskDir, "agent"); + mkdirSync(join(agentDir, "inputs"), { recursive: true }); + + const input = new ExcelJS.Workbook(); + const sheet = input.addWorksheet("Sheet1"); + sheet.getCell("A3").value = "A"; + sheet.getCell("A4").value = "B"; + sheet.getCell("I3").value = 0.2; + sheet.getCell("J3").value = 0.5; + sheet.getCell("B15").value = "A"; + sheet.getCell("B16").value = "B"; + sheet.getCell("H15").value = 0.3; + sheet.getCell("H16").value = 999; + await input.xlsx.writeFile(join(agentDir, "inputs", "input.xlsx")); + writeJson(join(agentDir, "task.json"), { + schema: 1, + taskId: "Template/lookup-bounds", + track: "spreadsheetbench-v2", + category: "Template", + instruction, + inputFiles: ["inputs/input.xlsx"], + promptFiles: [], + }); + return { agentManifest: join(agentDir, "task.json"), instruction }; +} + async function stagedDebtWaterfallTask(root: string): Promise<{ agentManifest: string }> { const taskDir = join(root, "tasks", "debt-waterfall"); const agentDir = join(taskDir, "agent"); diff --git a/tests/spreadsheetBenchOfficialV2Project.test.ts b/tests/spreadsheetBenchOfficialV2Project.test.ts new file mode 100644 index 00000000..f30352b6 --- /dev/null +++ b/tests/spreadsheetBenchOfficialV2Project.test.ts @@ -0,0 +1,1046 @@ +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { stableTraceHash } from "../src/nodeagent/traces"; + +const tempRoots: string[] = []; +const MODEL_NAME = "example/free:free"; +const MODEL_CALLS = 4; +const categoryCounts = { + Debugging: 100, + Financial_Model: 100, + Template: 97, + Visualization: 24, +}; + +type FileEvidence = { path: string; sha256: string; bytes: number }; + +type ArtifactRef = { + kind: string; + refId: string; + label: string; + hash: string; +}; + +type FrameTraceEvent = { + step: number; + tool: string; + args: Record; + result: Record; + ms: number; +}; + +type StageReceipt = { + traceId: string; + stage: string; + status: string; + attempts: number; + operationCount?: number; + summary: string; + events: Array<{ + traceId: string; + eventIndex: number; + step: number; + tool: string; + argsHash: string; + resultHash: string; + }>; +}; + +type MinimalNodeAgentTrace = { + schema: string; + traceId: string; + createdAt: number; + updatedAt: number; + trigger: { + kind: string; + prompt: string; + selectedArtifactIds: string[]; + openedSurface: string; + }; + plan: { + goal: string; + plannedReads: ArtifactRef[]; + plannedWrites: ArtifactRef[]; + approvalRequired: boolean; + riskFlags: string[]; + }; + contextPack: { + worldModelHash: string; + includedRefs: ArtifactRef[]; + excludedRefs: unknown[]; + }; + steps: Array>; + evidence: Array>; + mutations: Array>; + approvals: unknown[]; + eval: { + benchmarkCaseId: string; + proofArtifacts: ArtifactRef[]; + }; + final: { + outputArtifactRefs: ArtifactRef[]; + summary: string; + status: string; + }; +}; + +type MinimalNodeAgentReceipt = { + schema: string; + traceId: string; + taskId: string; + track: string; + category: string; + candidateWorkbookPath: string; + candidateWorkbookSha256: string; + outcome: { + status: string; + mutatingTask: boolean; + changedCellCount: number; + finalVerificationStatus: string; + }; + stages: Record; + isolation: { + boundary: string; + agentRoot: string; + openedAgentFiles: string[]; + evaluatorMetadataAccess: string; + evaluatorFileReadCount: number; + candidateEmittedBeforeEvaluatorAccess: boolean; + }; + model: { + name: string; + calls: number; + usage: { inputTokens: number; outputTokens: number }; + }; + recalculation: { + engine: string; + attemptedFormulaCount: number; + refreshedFormulaCount: number; + unresolvedFormulaCount: number; + unresolved: unknown[]; + }; + frame: { + status: string; + agentResult: { + stopReason: string; + usage: { + modelCalls: number; + inputTokens: number; + outputTokens: number; + }; + trace: FrameTraceEvent[]; + }; + }; + trace: MinimalNodeAgentTrace; +}; + +type ProjectionResult = { + taskId: string; + category: string; + mode: string; + candidateWorkbook: string; + model: { name: string; calls: number }; + sidecarEvidence: { + nodeAgentReceipt: FileEvidence; + nodeAgentTrace: FileEvidence; + }; +}; + +type ProjectionFixture = { + root: string; + runRoot: string; + datasetRoot: string; + upstream: string; + outputs: string; + reportPath: string; + receiptPath: string; + report: { + schema: number; + mode: string; + taskCount: number; + harness: { toolPolicy: string; evaluatorAccess: string }; + results: ProjectionResult[]; + }; +}; + +afterEach(() => { + for (const root of tempRoots.splice(0)) + rmSync(root, { recursive: true, force: true }); +}); + +describe("SpreadsheetBench V2 official projection", () => { + it("accepts authentic minimal NodeAgent receipts and traces bound to every candidate", () => { + const fixture = createFixture(); + const wordingClassifiedNonmutating = fixture.report.results.find( + (result) => result.taskId === "Debugging/001", + )!; + rewriteSidecars( + fixture, + wordingClassifiedNonmutating, + (receipt) => { + receipt.outcome.mutatingTask = false; + }, + ); + const terminalAfterProof = fixture.report.results.find( + (result) => result.taskId === "Debugging/002", + )!; + rewriteSidecars(fixture, terminalAfterProof, (receipt) => { + receipt.frame.status = "blocked"; + receipt.frame.agentResult.stopReason = "step_budget"; + }); + writeJson(fixture.reportPath, fixture.report); + const evaluatorBefore = readFileSync( + join(fixture.upstream, "evaluation", "evaluation.py"), + "utf8", + ); + const visualEvaluatorBefore = readFileSync( + join(fixture.upstream, "evaluation", "run_visual_vlm_checklist_eval.py"), + "utf8", + ); + + const accepted = runProjection(fixture); + + expect(accepted.status, `${accepted.stdout}\n${accepted.stderr}`).toBe(0); + const projection = readJson<{ + harnessMode: string; + taskCount: number; + projectedOutputCount: number; + projectionErrorCount: number; + cases: Array<{ + taskId: string; + sourceSha256: string; + outputSha256: string; + nodeAgentReceipt: FileEvidence; + nodeAgentTrace: FileEvidence; + }>; + }>(fixture.receiptPath); + expect(projection).toMatchObject({ + harnessMode: "nodeagent-workbook", + taskCount: 321, + projectedOutputCount: 321, + projectionErrorCount: 0, + }); + expect(projection.cases[0]).toMatchObject({ + taskId: "Debugging/001", + nodeAgentReceipt: + fixture.report.results[0].sidecarEvidence.nodeAgentReceipt, + nodeAgentTrace: fixture.report.results[0].sidecarEvidence.nodeAgentTrace, + }); + expect( + projection.cases.every((item) => item.sourceSha256 === item.outputSha256), + ).toBe(true); + expect( + readFileSync( + join(fixture.upstream, "evaluation", "evaluation.py"), + "utf8", + ), + ).toBe(evaluatorBefore); + expect( + readFileSync( + join( + fixture.upstream, + "evaluation", + "run_visual_vlm_checklist_eval.py", + ), + "utf8", + ), + ).toBe(visualEvaluatorBefore); + }, 20_000); + + it("rejects semantically forged sidecars even when their declared hashes are recomputed", () => { + const fixture = createFixture(); + const mismatches: Array<{ + index: number; + error: string; + mutate: ( + receipt: MinimalNodeAgentReceipt, + trace: MinimalNodeAgentTrace, + result: ProjectionResult, + ) => void; + }> = [ + { + index: 0, + error: "NodeAgent receipt schema mismatch", + mutate: (receipt) => { + receipt.schema = "noderoom.spreadsheetbench.nodeagent_bridge.v0"; + }, + }, + { + index: 1, + error: "NodeAgent receipt taskId mismatch", + mutate: (receipt) => { + receipt.taskId = "Debugging/forged"; + }, + }, + { + index: 2, + error: "NodeAgent receipt traceId is missing", + mutate: (receipt) => { + receipt.traceId = ""; + }, + }, + { + index: 3, + error: "NodeAgent receipt candidate workbook hash mismatch", + mutate: (receipt) => { + receipt.candidateWorkbookSha256 = "0".repeat(64); + }, + }, + { + index: 4, + error: "NodeAgent receipt isolation/evaluator-access contract mismatch", + mutate: (receipt) => { + receipt.isolation.evaluatorMetadataAccess = "opened_before_candidate"; + }, + }, + { + index: 5, + error: "NodeAgent receipt model name mismatch", + mutate: (receipt) => { + receipt.model.name = "forged/model"; + }, + }, + { + index: 6, + error: "NodeAgent receipt model calls mismatch", + mutate: (receipt) => { + receipt.model.calls += 1; + }, + }, + { + index: 7, + error: "NodeAgent trace schema mismatch", + mutate: (receipt, trace) => { + receipt.trace.schema = "nodeagent.trace.v0"; + trace.schema = "nodeagent.trace.v0"; + }, + }, + { + index: 8, + error: "NodeAgent traceId mismatch", + mutate: (receipt, trace) => { + receipt.trace.traceId = "spreadsheetbench-nodeagent-forged"; + trace.traceId = "spreadsheetbench-nodeagent-forged"; + }, + }, + { + index: 9, + error: "NodeAgent trace benchmark case mismatch", + mutate: (receipt, trace) => { + receipt.trace.eval.benchmarkCaseId = "Debugging/forged"; + trace.eval.benchmarkCaseId = "Debugging/forged"; + }, + }, + { + index: 10, + error: "NodeAgent trace candidate proof hash mismatch", + mutate: (receipt, trace) => { + receipt.trace.eval.proofArtifacts[0].hash = "1".repeat(64); + trace.eval.proofArtifacts[0].hash = "1".repeat(64); + }, + }, + { + index: 11, + error: "NodeAgent trace final artifact hash mismatch", + mutate: (receipt, trace) => { + receipt.trace.final.outputArtifactRefs[0].hash = "2".repeat(64); + trace.final.outputArtifactRefs[0].hash = "2".repeat(64); + }, + }, + { + index: 12, + error: "NodeAgent receipt embedded trace does not match trace sidecar", + mutate: (receipt) => { + receipt.trace.final.summary = "forged embedded summary"; + }, + }, + { + index: 13, + error: "NodeAgent frame/model usage mismatch", + mutate: (receipt) => { + receipt.frame.agentResult.usage.modelCalls += 1; + }, + }, + { + index: 14, + error: "NodeAgent frame execution receipt is invalid", + mutate: (receipt) => { + receipt.frame.agentResult.trace = []; + }, + }, + { + index: 15, + error: "NodeAgent plan stage receipt is invalid", + mutate: (receipt) => { + receipt.stages.plan.attempts += 1; + }, + }, + { + index: 16, + error: "NodeAgent verify stage event does not bind its frame event", + mutate: (receipt) => { + receipt.stages.verify.events[0].resultHash = "fnv1a:00000000"; + }, + }, + { + index: 17, + error: "NodeAgent trace evidence is missing", + mutate: (receipt, trace) => { + receipt.trace.evidence = []; + trace.evidence = []; + }, + }, + { + index: 18, + error: "NodeAgent trace mutations do not match frame writes", + mutate: (receipt, trace) => { + receipt.trace.mutations = []; + trace.mutations = []; + }, + }, + { + index: 19, + error: "NodeAgent verify stage status does not match frame events", + mutate: (receipt) => { + receipt.stages.verify.status = "needs_repair"; + }, + }, + { + index: 20, + error: "NodeAgent inspect stage events do not match frame semantics", + mutate: (receipt) => { + receipt.stages.inspect.events = structuredClone( + receipt.stages.plan.events, + ); + receipt.stages.inspect.attempts = receipt.stages.inspect.events.length; + }, + }, + { + index: 23, + error: "NodeAgent changed cell count does not match committed mutations", + mutate: (receipt) => { + receipt.outcome.mutatingTask = false; + receipt.outcome.changedCellCount = 0; + }, + }, + { + index: 24, + error: "NodeAgent inspect stage status does not match frame events", + mutate: (receipt, trace) => { + receipt.frame.agentResult.trace[0].result = { + ok: false, + error: "inspection failed", + }; + rebindFrameEvent(receipt, trace, 0); + }, + }, + { + index: 25, + error: "NodeAgent verify stage status does not match frame events", + mutate: (receipt, trace) => { + const event = receipt.frame.agentResult.trace.at(-1)!; + const phases = event.result.phases as Record>; + phases.verify.status = "needs_repair"; + rebindFrameEvent( + receipt, + trace, + receipt.frame.agentResult.trace.length - 1, + ); + }, + }, + { + index: 26, + error: "NodeAgent write operation count does not match frame events", + mutate: (receipt) => { + receipt.stages.write.operationCount! += 1; + }, + }, + ]; + const expectedErrors: string[] = []; + for (const mismatch of mismatches) { + const result = fixture.report.results[mismatch.index]; + rewriteSidecars(fixture, result, mismatch.mutate); + expectedErrors.push(`${mismatch.error}: ${result.taskId}`); + } + const debugging = fixture.report.results.find( + (result) => result.taskId === "Debugging/022", + )!; + const financialModel = fixture.report.results.find( + (result) => result.taskId === "Financial_Model/022", + )!; + debugging.category = "Financial_Model"; + financialModel.category = "Debugging"; + expectedErrors.push( + "task category conflicts with taskId: Debugging/022 declares Financial_Model", + "task category conflicts with taskId: Financial_Model/022 declares Debugging", + ); + const wrongDirectory = fixture.report.results.find( + (result) => result.taskId === "Debugging/023", + )!; + const otherDirectory = fixture.report.results.find( + (result) => result.taskId === "Financial_Model/023", + )!; + wrongDirectory.sidecarEvidence.nodeAgentReceipt = structuredClone( + otherDirectory.sidecarEvidence.nodeAgentReceipt, + ); + expectedErrors.push( + "NodeAgent receipt path is not canonical for the candidate: Debugging/023", + ); + writeJson(fixture.reportPath, fixture.report); + + const rejected = runProjection(fixture); + + expect(rejected.status, `${rejected.stdout}\n${rejected.stderr}`).toBe(1); + const projection = readJson<{ + projectedOutputCount: number; + errors: string[]; + }>(fixture.receiptPath); + expect(projection.projectedOutputCount).toBe( + 321 - mismatches.length - 3, + ); + for (const error of expectedErrors) + expect(projection.errors).toContain(error); + }, 20_000); + + it("rejects a report that does not preserve evaluator-after-candidate access", () => { + const fixture = createHarnessContractFixture(); + + const rejected = runProjection(fixture); + + expect(rejected.status).toBe(1); + expect(rejected.stderr).toContain( + "nodeagent-workbook report harness must declare toolPolicy=agent_dir_only_until_candidate and evaluatorAccess=after_candidate_emit_only", + ); + }, 20_000); +}); + +function createFixture(): ProjectionFixture { + const root = mkdtempSync(join(tmpdir(), "spreadsheetbench-v2-project-")); + tempRoots.push(root); + const runRoot = join(root, "run"); + const datasetRoot = join(root, "dataset"); + const upstream = join(root, "upstream"); + const outputs = join(root, "official-outputs"); + const reportPath = join(root, "report.json"); + const receiptPath = join(root, "projection.json"); + const results: ProjectionResult[] = []; + + for (const [category, count] of Object.entries(categoryCounts)) { + const tasks = Array.from({ length: count }, (_, index) => ({ + id: String(index + 1).padStart(3, "0"), + })); + writeJson(join(datasetRoot, category, "dataset.json"), tasks); + for (const task of tasks) { + const taskId = `${category}/${task.id}`; + const taskDir = `${category}_${task.id}/attempt-01`; + const candidateWorkbook = `${taskDir}/candidate-${task.id}_input.xlsx`; + const candidatePath = join(runRoot, candidateWorkbook); + writeFile(candidatePath, `candidate:${taskId}`); + const candidateSha256 = sha256(readFileSync(candidatePath)); + const traceId = `spreadsheetbench-nodeagent-${sha256(taskId).slice(0, 12)}`; + const frameTrace = minimalFrameTrace(); + const trace = minimalTrace({ + taskId, + traceId, + candidatePath, + candidateSha256, + frameTrace, + }); + const receipt = minimalReceipt({ + taskId, + category, + traceId, + candidatePath, + candidateSha256, + frameTrace, + trace, + }); + results.push({ + taskId, + category, + mode: "nodeagent-workbook", + candidateWorkbook, + model: { name: MODEL_NAME, calls: MODEL_CALLS }, + sidecarEvidence: { + nodeAgentReceipt: writeJsonEvidence( + runRoot, + `${taskDir}/nodeagent-workbook-receipt.json`, + receipt, + ), + nodeAgentTrace: writeJsonEvidence( + runRoot, + `${taskDir}/nodeagent-workbook-trace.json`, + trace, + ), + }, + }); + } + } + + writeFile( + join(upstream, "evaluation", "evaluation.py"), + "# immutable deterministic evaluator\n", + ); + writeFile( + join(upstream, "evaluation", "run_visual_vlm_checklist_eval.py"), + "# immutable visual evaluator\n", + ); + writeFile( + join(upstream, ".git", "HEAD"), + "0123456789abcdef0123456789abcdef01234567\n", + ); + const report = { + schema: 1, + mode: "nodeagent-workbook", + taskCount: 321, + harness: { + toolPolicy: "agent_dir_only_until_candidate", + evaluatorAccess: "after_candidate_emit_only", + }, + results, + }; + writeJson(reportPath, report); + return { + root, + runRoot, + datasetRoot, + upstream, + outputs, + reportPath, + receiptPath, + report, + }; +} + +function createHarnessContractFixture(): ProjectionFixture { + const root = mkdtempSync(join(tmpdir(), "spreadsheetbench-v2-project-contract-")); + tempRoots.push(root); + const reportPath = join(root, "report.json"); + const report: ProjectionFixture["report"] = { + schema: 1, + mode: "nodeagent-workbook", + taskCount: 321, + harness: { + toolPolicy: "agent_dir_only_until_candidate", + evaluatorAccess: "available_during_candidate_generation", + }, + results: Array.from({ length: 321 }, () => ({}) as ProjectionResult), + }; + writeJson(reportPath, report); + return { + root, + runRoot: join(root, "run"), + datasetRoot: join(root, "dataset"), + upstream: join(root, "upstream"), + outputs: join(root, "official-outputs"), + reportPath, + receiptPath: join(root, "projection.json"), + report, + }; +} + +function minimalFrameTrace(): FrameTraceEvent[] { + return [ + { + step: 0, + tool: "inspect_workbook", + args: { artifactId: "Sheet1", instruction: "Complete the workbook." }, + result: { ok: true, artifactId: "Sheet1", inspectedCellCount: 12 }, + ms: 4, + }, + { + step: 1, + tool: "execute_verified_workbook_plan", + args: { artifactId: "Sheet1", instruction: "Complete the workbook." }, + result: { + ok: true, + status: "completed", + operationCount: 1, + changedTargetCount: 1, + phases: { + plan: { status: "completed", targets: ["A1"] }, + preflight: { status: "passed" }, + write: { status: "completed" }, + verify: { status: "passed" }, + }, + }, + ms: 7, + }, + { + step: 2, + tool: "execute_verified_workbook_plan", + args: { artifactId: "Sheet1", instruction: "Repair the workbook." }, + result: { + ok: true, + status: "completed", + operationCount: 1, + changedTargetCount: 1, + phases: { + plan: { status: "completed", targets: ["A1"] }, + preflight: { status: "passed" }, + write: { status: "completed" }, + verify: { status: "passed" }, + }, + }, + ms: 5, + }, + ]; +} + +function minimalTrace(args: { + taskId: string; + traceId: string; + candidatePath: string; + candidateSha256: string; + frameTrace: FrameTraceEvent[]; +}): MinimalNodeAgentTrace { + const candidateRef: ArtifactRef = { + kind: "artifact", + refId: args.candidatePath, + label: `SpreadsheetBench candidate ${args.taskId}`, + hash: args.candidateSha256, + }; + return { + schema: "nodeagent.trace.v1", + traceId: args.traceId, + createdAt: 1_000, + updatedAt: 1_001, + trigger: { + kind: "benchmark", + prompt: `Complete ${args.taskId}`, + selectedArtifactIds: ["Sheet1"], + openedSurface: "spreadsheetbench.nodeagent.bridge", + }, + plan: { + goal: `Complete ${args.taskId}`, + plannedReads: [], + plannedWrites: [structuredClone(candidateRef)], + approvalRequired: false, + riskFlags: [ + "evaluator_isolation", + "workbook_mutation", + "cas_managed_write", + ], + }, + contextPack: { + worldModelHash: "fnv1a:test-world-model", + includedRefs: [], + excludedRefs: [], + }, + steps: args.frameTrace.map((event, eventIndex) => { + const argsHash = stableTraceHash(event.args); + const resultHash = stableTraceHash(event.result); + const stepId = `${args.traceId}:tool:${String(event.step).padStart(3, "0")}:${event.tool.replaceAll("_", "-")}`; + return { + stepId, + traceId: args.traceId, + phase: "tool_call", + title: `Tool call: ${event.tool}`, + summary: `Recorded frame event ${eventIndex}.`, + inputRefs: [ + { + kind: "tool_result", + refId: `${stepId}:args`, + hash: argsHash, + }, + ], + outputRefs: [ + { + kind: "tool_result", + refId: `${stepId}:result`, + hash: resultHash, + }, + ], + tool: { + name: event.tool, + argsHash, + resultHash, + status: "ok", + latencyMs: event.ms, + }, + timings: { + startedAt: 1_000 + eventIndex, + endedAt: 1_000 + eventIndex + event.ms, + latencyMs: event.ms, + }, + verdict: { status: "ok" }, + }; + }), + evidence: [ + { + receiptId: `${args.traceId}:evidence:spreadsheetbench-inspect`, + traceId: args.traceId, + label: "SpreadsheetBench inspect", + sourceRefs: [ + { + kind: "tool_result", + refId: `${args.traceId}:bridge:event:0`, + hash: stableTraceHash(args.frameTrace[0].result), + }, + ], + artifactRefs: [structuredClone(candidateRef)], + factHash: stableTraceHash({ status: "completed" }), + verifier: "spreadsheetBenchNodeAgentBridge", + status: "verified", + }, + { + receiptId: `${args.traceId}:evidence:spreadsheetbench-verify`, + traceId: args.traceId, + label: "SpreadsheetBench verify", + sourceRefs: [ + { + kind: "tool_result", + refId: `${args.traceId}:bridge:event:${args.frameTrace.length - 1}`, + hash: stableTraceHash(args.frameTrace.at(-1)!.result), + }, + ], + artifactRefs: [structuredClone(candidateRef)], + factHash: stableTraceHash({ status: "completed" }), + verifier: "verify_workbook", + status: "verified", + }, + ], + mutations: args.frameTrace.slice(1).map((event) => ({ + receiptId: `${args.traceId}:mutation:${stableTraceHash(event.args).slice(-8)}`, + traceId: args.traceId, + targetRefs: [ + { + kind: "cell", + refId: "Sheet1!A1", + label: "SpreadsheetBench workbook target", + }, + ], + payloadHash: stableTraceHash(event.args), + status: "committed", + })), + approvals: [], + eval: { + benchmarkCaseId: args.taskId, + proofArtifacts: [structuredClone(candidateRef)], + }, + final: { + outputArtifactRefs: [structuredClone(candidateRef)], + summary: "Workbook candidate emitted.", + status: "completed", + }, + }; +} + +function minimalReceipt(args: { + taskId: string; + category: string; + traceId: string; + candidatePath: string; + candidateSha256: string; + frameTrace: FrameTraceEvent[]; + trace: MinimalNodeAgentTrace; +}): MinimalNodeAgentReceipt { + const eventRef = (eventIndex: number) => { + const event = args.frameTrace[eventIndex]; + return { + traceId: args.traceId, + eventIndex, + step: event.step, + tool: event.tool, + argsHash: stableTraceHash(event.args), + resultHash: stableTraceHash(event.result), + }; + }; + const stageReceipt = ( + stage: string, + eventIndexes: number[] = [], + operationCount?: number, + ): StageReceipt => ({ + traceId: args.traceId, + stage, + status: eventIndexes.length === 0 ? "skipped" : "completed", + attempts: eventIndexes.length, + ...(operationCount === undefined ? {} : { operationCount }), + summary: eventIndexes.length === 0 ? `${stage} skipped` : `${stage} complete`, + events: eventIndexes.map(eventRef), + }); + const stages: Record = { + inspect: stageReceipt("inspect", [0]), + plan: stageReceipt("plan", [1, 2], 1), + preflight: stageReceipt("preflight", [1, 2], 1), + write: stageReceipt("write", [1, 2], 2), + verify: stageReceipt("verify", [1, 2], 1), + repair: stageReceipt("repair"), + }; + return { + schema: "noderoom.spreadsheetbench.nodeagent_bridge.v1", + traceId: args.traceId, + taskId: args.taskId, + track: "spreadsheetbench-v2", + category: args.category, + candidateWorkbookPath: args.candidatePath, + candidateWorkbookSha256: args.candidateSha256, + outcome: { + status: "completed", + mutatingTask: true, + changedCellCount: 2, + finalVerificationStatus: "passed", + }, + stages, + isolation: { + boundary: "agent_visible_files_only", + agentRoot: join(dirname(args.candidatePath), "agent-workspace", "agent"), + openedAgentFiles: ["task.json", "inputs/input.xlsx"], + evaluatorMetadataAccess: "none", + evaluatorFileReadCount: 0, + candidateEmittedBeforeEvaluatorAccess: true, + }, + model: { + name: MODEL_NAME, + calls: MODEL_CALLS, + usage: { inputTokens: 120, outputTokens: 24 }, + }, + recalculation: { + engine: "nodeagent-formula-engine", + attemptedFormulaCount: 0, + refreshedFormulaCount: 0, + unresolvedFormulaCount: 0, + unresolved: [], + }, + frame: { + status: "completed", + agentResult: { + stopReason: "done", + usage: { modelCalls: MODEL_CALLS, inputTokens: 120, outputTokens: 24 }, + trace: structuredClone(args.frameTrace), + }, + }, + trace: structuredClone(args.trace), + }; +} + +function rewriteSidecars( + fixture: ProjectionFixture, + result: ProjectionResult, + mutate: ( + receipt: MinimalNodeAgentReceipt, + trace: MinimalNodeAgentTrace, + result: ProjectionResult, + ) => void, +): void { + const receiptPath = join( + fixture.runRoot, + result.sidecarEvidence.nodeAgentReceipt.path, + ); + const tracePath = join( + fixture.runRoot, + result.sidecarEvidence.nodeAgentTrace.path, + ); + const receipt = readJson(receiptPath); + const trace = readJson(tracePath); + mutate(receipt, trace, result); + result.sidecarEvidence.nodeAgentReceipt = writeJsonEvidence( + fixture.runRoot, + result.sidecarEvidence.nodeAgentReceipt.path, + receipt, + ); + result.sidecarEvidence.nodeAgentTrace = writeJsonEvidence( + fixture.runRoot, + result.sidecarEvidence.nodeAgentTrace.path, + trace, + ); +} + +function rebindFrameEvent( + receipt: MinimalNodeAgentReceipt, + trace: MinimalNodeAgentTrace, + eventIndex: number, +): void { + const event = receipt.frame.agentResult.trace[eventIndex]; + const resultHash = stableTraceHash(event.result); + for (const stage of Object.values(receipt.stages)) { + for (const eventRef of stage.events) { + if (eventRef.eventIndex === eventIndex) eventRef.resultHash = resultHash; + } + } + const failed = + event.result.ok === false || typeof event.result.error === "string"; + for (const target of [receipt.trace, trace]) { + const step = target.steps[eventIndex]; + const tool = step.tool as Record; + tool.resultHash = resultHash; + tool.status = failed ? "failed" : "ok"; + const outputRefs = step.outputRefs as Array>; + for (const ref of outputRefs) ref.hash = resultHash; + const verdict = step.verdict as Record; + verdict.status = failed ? "failed" : "ok"; + for (const evidence of target.evidence) { + const sourceRefs = evidence.sourceRefs as Array>; + for (const ref of sourceRefs) { + if (ref.refId === `${receipt.traceId}:bridge:event:${eventIndex}`) { + ref.hash = resultHash; + } + } + } + } +} + +function runProjection(fixture: ProjectionFixture) { + return spawnSync( + process.execPath, + [ + resolve("node_modules", "tsx", "dist", "cli.mjs"), + resolve("scripts", "spreadsheetbench-official-v2-project.ts"), + "--report", + fixture.reportPath, + "--run-root", + fixture.runRoot, + "--dataset-root", + fixture.datasetRoot, + "--upstream-repo", + fixture.upstream, + "--outputs-root", + fixture.outputs, + "--receipt-out", + fixture.receiptPath, + "--clean", + ], + { cwd: resolve("."), encoding: "utf8" }, + ); +} + +function writeJsonEvidence( + root: string, + relativePath: string, + value: unknown, +): FileEvidence { + const content = `${JSON.stringify(value, null, 2)}\n`; + writeFile(join(root, relativePath), content); + return { + path: relativePath, + sha256: sha256(content), + bytes: Buffer.byteLength(content), + }; +} + +function sha256(value: string | Buffer): string { + return createHash("sha256").update(value).digest("hex"); +} + +function readJson(path: string): T { + return JSON.parse(readFileSync(path, "utf8")) as T; +} + +function writeJson(path: string, value: unknown): void { + writeFile(path, `${JSON.stringify(value, null, 2)}\n`); +} + +function writeFile(path: string, content: string): void { + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, content, "utf8"); +} diff --git a/tests/workbookTaskIntelligence.test.ts b/tests/workbookTaskIntelligence.test.ts index 69ebb886..0fb67d97 100644 --- a/tests/workbookTaskIntelligence.test.ts +++ b/tests/workbookTaskIntelligence.test.ts @@ -26,6 +26,21 @@ describe("workbook task intelligence", () => { expect(references.find((reference) => reference.start === "J15")?.role).toBe("target"); }); + it("classifies user inputs and lookup bounds as dependencies while keeping the requested output range as the target", () => { + const instruction = "I want to configure column 'J' under 'Densities' in my Excel sheet to return 'PASS' or 'FAIL' similar to what I have set up for column 'I'. For column 'I', I've used the formula =IF(AND(G15<1.05,G15>0.95),'PASS','FAIL'). In the 'Densities' table, the user provides two inputs: they select a variable from 'A-G' in cells B15:B17, and they enter a reading in cells H15:H17. I need column J (cells J15:J17) to check the corresponding cell in column B (B15:B17), and then verify if the moisture range entered by the user falls within a specific range defined in cells 'I3' and 'J3'. In cell I15, I have a pass/fail condition. Adjust the calculation based on the user's choice from a dropdown that populates B3:B9."; + const references = extractWorkbookTaskReferences(instruction, ["Sheet1"]); + const roles = new Map(references.map((reference) => [`${reference.start}:${reference.end}`, reference.role])); + + expect(roles.get("G15:G15")).toBe("dependency"); + expect(roles.get("B15:B17")).toBe("dependency"); + expect(roles.get("H15:H17")).toBe("dependency"); + expect(roles.get("J15:J17")).toBe("target"); + expect(roles.get("I3:I3")).toBe("dependency"); + expect(roles.get("J3:J3")).toBe("dependency"); + expect(roles.get("I15:I15")).toBe("dependency"); + expect(roles.get("B3:B9")).toBe("dependency"); + }); + it("selects the quoted formula cell and its input even under a starved snapshot cap", () => { const cells: WorkbookObservedCell[] = [ ...Array.from({ length: 20 }, (_, index) => ({ sheet: "ATTENDENCE", address: `${String.fromCharCode(65 + index)}2`, value: "period" })), @@ -284,7 +299,7 @@ describe("workbook task intelligence", () => { ])); }); - it("requires complete coverage of a visible weekday formula-fill band", () => { + it("uses populated weekday labels as evidence and repairs only the formula anchor", () => { const cells: WorkbookObservedCell[] = [ { sheet: "ATTENDENCE", address: "F3", value: "21", formula: 'TEXT(F4,"DD")' }, { sheet: "ATTENDENCE", address: "F4", value: "2015-10-21" }, @@ -295,22 +310,114 @@ describe("workbook task intelligence", () => { { sheet: "ATTENDENCE", address: "I3", value: "S" }, { sheet: "ATTENDENCE", address: "I4", value: "2015-10-24" }, ]; - const instruction = 'Correct the weekday formula TEXT(F4,"DD") so the weekday names display like Mon and Wed.'; + const instruction = 'Correct only the wrong weekday formula TEXT(F4,"DD") and preserve the other existing weekday labels.'; const inspection = inspectWorkbookTask({ instruction, sheetNames: ["ATTENDENCE"], cells }); expect(inspection.findings).toEqual(expect.arrayContaining([ - expect.objectContaining({ kind: "formula_fill_band", address: "F3", relatedAddresses: ["G3", "H3", "I3"] }), + expect.objectContaining({ kind: "formula_fill_band", address: "F3", relatedAddresses: [] }), ])); expect(inspection.formulaFillSuggestions).toEqual([expect.objectContaining({ - range: "F3:I3", + range: "F3:F3", sourceFormula: 'TEXT(F4,"DDD")', operations: [ { sheet: "ATTENDENCE", cell: "F3", formula: 'TEXT(F4,"DDD")' }, - { sheet: "ATTENDENCE", cell: "G3", formula: 'TEXT(G4,"DDD")' }, - { sheet: "ATTENDENCE", cell: "H3", formula: 'TEXT(H4,"DDD")' }, - { sheet: "ATTENDENCE", cell: "I3", formula: 'TEXT(I4,"DDD")' }, ], })]); + const repaired = verifyWorkbookPlan({ + instruction, + inspection, + cells, + sheetNames: ["ATTENDENCE"], + operations: [{ sheet: "ATTENDENCE", cell: "F3", formula: 'TEXT(F4,"DDD")' }], + }); + expect(repaired.status).toBe("passed"); + expect(repaired.checks).toMatchObject({ targetCandidateCount: 1, coveredTargetCount: 1 }); + }); + + it("preserves correct peer weekday formulas and repairs only the quoted anchor", () => { + const cells: WorkbookObservedCell[] = [ + { sheet: "ATTENDENCE", address: "F3", value: "21", formula: 'TEXT(F4,"DD")' }, + { sheet: "ATTENDENCE", address: "F4", value: "2015-10-21" }, + { sheet: "ATTENDENCE", address: "G3", value: "Thursday", formula: 'TEXT(G4,"DDDD")' }, + { sheet: "ATTENDENCE", address: "G4", value: "2015-10-22" }, + { sheet: "ATTENDENCE", address: "H3", value: "Friday", formula: 'TEXT(H4,"DDDD")' }, + { sheet: "ATTENDENCE", address: "H4", value: "2015-10-23" }, + ]; + const instruction = 'Correct only the wrong weekday formula TEXT(F4,"DD") and preserve the other existing weekday formulas.'; + const inspection = inspectWorkbookTask({ instruction, sheetNames: ["ATTENDENCE"], cells }); + const plan = buildWorkbookSuggestedPlan(inspection, "ATTENDENCE"); + + expect(plan).toEqual({ + conflicts: [], + operations: [{ elementId: "F3", formula: 'TEXT(F4,"DDDD")' }], + }); + expect(inspection.formulaRepairSuggestions).not.toEqual(expect.arrayContaining([ + expect.objectContaining({ cell: "F3" }), + expect.objectContaining({ cell: "G3" }), + expect.objectContaining({ cell: "H3" }), + ])); + expect(verifyWorkbookPlan({ + instruction, + inspection, + cells, + sheetNames: ["ATTENDENCE"], + operations: [{ sheet: "ATTENDENCE", cell: "F3", formula: 'TEXT(F4,"DDDD")' }], + }).status).toBe("passed"); + }); + + it("normalizes the full visible weekday row when canonical three-letter examples are requested", () => { + const cells: WorkbookObservedCell[] = [ + { sheet: "ATTENDENCE", address: "F3", value: "21", formula: 'TEXT(F4,"DD")' }, + { sheet: "ATTENDENCE", address: "F4", value: "2015-10-21" }, + { sheet: "ATTENDENCE", address: "G3", value: "TH" }, + { sheet: "ATTENDENCE", address: "G4", value: "2015-10-22" }, + { sheet: "ATTENDENCE", address: "H3", value: "F" }, + { sheet: "ATTENDENCE", address: "H4", value: "2015-10-23" }, + { sheet: "ATTENDENCE", address: "I3", value: "S" }, + { sheet: "ATTENDENCE", address: "I4", value: "2015-10-24" }, + ]; + const instruction = 'Correct the weekday formula TEXT(F4,"DD") so weekday names display like Mon and Wed.'; + const inspection = inspectWorkbookTask({ instruction, sheetNames: ["ATTENDENCE"], cells }); + + expect(inspection.formulaFillSuggestions).toEqual([expect.objectContaining({ + range: "F3:I3", + operations: ["F", "G", "H", "I"].map((column) => ({ + sheet: "ATTENDENCE", + cell: `${column}3`, + formula: `TEXT(${column}4,"DDD")`, + })), + })]); + expect(verifyWorkbookPlan({ + instruction, + inspection, + cells, + sheetNames: ["ATTENDENCE"], + operations: [{ sheet: "ATTENDENCE", cell: "F3", formula: 'TEXT(F4,"DDD")' }], + }).status).toBe("needs_repair"); + }); + + it("fills blank weekday peers but still requires complete coverage of those blank targets", () => { + const cells: WorkbookObservedCell[] = [ + { sheet: "ATTENDENCE", address: "F3", value: "21", formula: 'TEXT(F4,"DD")' }, + { sheet: "ATTENDENCE", address: "F4", value: "2015-10-21" }, + { sheet: "ATTENDENCE", address: "G3", value: "" }, + { sheet: "ATTENDENCE", address: "G4", value: "2015-10-22" }, + { sheet: "ATTENDENCE", address: "H3", value: "" }, + { sheet: "ATTENDENCE", address: "H4", value: "2015-10-23" }, + { sheet: "ATTENDENCE", address: "I3", value: "" }, + { sheet: "ATTENDENCE", address: "I4", value: "2015-10-24" }, + ]; + const instruction = 'Correct and fill the weekday formula TEXT(F4,"DD") so weekday names display like Mon and Wed.'; + const inspection = inspectWorkbookTask({ instruction, sheetNames: ["ATTENDENCE"], cells }); + + expect(inspection.formulaFillSuggestions).toEqual([expect.objectContaining({ + range: "F3:I3", + operations: ["F", "G", "H", "I"].map((column) => ({ + sheet: "ATTENDENCE", + cell: `${column}3`, + formula: `TEXT(${column}4,"DDD")`, + })), + })]); const partial = verifyWorkbookPlan({ instruction, inspection, @@ -335,6 +442,133 @@ describe("workbook task intelligence", () => { expect(complete.status).toBe("passed"); }); + it("derives a visible lookup-range pass/fail contract without treating source inputs as write targets", () => { + const instruction = "In the Densities table, users select a variable from A-G in cells B15:B17 and enter a reading in cells H15:H17. I need column J (cells J15:J17) to check the corresponding cell in column B (B15:B17) and verify whether the moisture falls within the range defined in cells I3 and J3. Display Pass when it is in range and Fail otherwise. The dropdown populates B3:B9."; + const cells: WorkbookObservedCell[] = [ + ...["A", "B", "C", "D", "E", "F", "G"].map((value, index) => ({ sheet: "Sheet1", address: `A${index + 3}`, value })), + ...Array.from({ length: 7 }, (_, index) => [ + { sheet: "Sheet1", address: `I${index + 3}`, value: 0.2 + index / 100 }, + { sheet: "Sheet1", address: `J${index + 3}`, value: 0.5 + index / 100 }, + ]).flat(), + { sheet: "Sheet1", address: "B15", value: "a" }, + { sheet: "Sheet1", address: "B16", value: "B" }, + { sheet: "Sheet1", address: "B17", value: "a" }, + { sheet: "Sheet1", address: "H15", value: 0.44 }, + { sheet: "Sheet1", address: "H16", value: 0.3 }, + { sheet: "Sheet1", address: "H17", value: 0.18 }, + { sheet: "Sheet1", address: "J15", value: "" }, + { sheet: "Sheet1", address: "J16", value: "" }, + { sheet: "Sheet1", address: "J17", value: "" }, + ]; + const inspection = inspectWorkbookTask({ instruction, sheetNames: ["Sheet1"], cells }); + const plan = buildWorkbookSuggestedPlan(inspection, "Sheet1"); + + expect(inspection.targetCandidates.map((target) => target.address)).toEqual(["J15", "J16", "J17"]); + expect(inspection.dependencyCandidates.map((dependency) => dependency.address)).toEqual(expect.arrayContaining([ + "B15", "B16", "B17", "H15", "H16", "H17", "I3", "J3", + ])); + expect(plan).toEqual({ + conflicts: [], + operations: [ + { elementId: "J15", formula: 'IF(MEDIAN(H15,VLOOKUP(B15,$A$3:$J$9,{9,10},0))=H15,"Pass","Fail")' }, + { elementId: "J16", formula: 'IF(MEDIAN(H16,VLOOKUP(B16,$A$3:$J$9,{9,10},0))=H16,"Pass","Fail")' }, + { elementId: "J17", formula: 'IF(MEDIAN(H17,VLOOKUP(B17,$A$3:$J$9,{9,10},0))=H17,"Pass","Fail")' }, + ], + }); + }); + + it("does not infer pass/fail formulas when any selected lookup key has missing bounds", () => { + const instruction = "Users select A-B in B15:B16 and enter readings in H15:H16. Fill J15:J16 with Pass or Fail using the corresponding lookup range defined by I3 and J3. The dropdown populates B3:B4."; + const cells: WorkbookObservedCell[] = [ + { sheet: "Sheet1", address: "A3", value: "A" }, + { sheet: "Sheet1", address: "A4", value: "B" }, + { sheet: "Sheet1", address: "I3", value: 0.2 }, + { sheet: "Sheet1", address: "J3", value: 0.5 }, + { sheet: "Sheet1", address: "I4", value: "" }, + { sheet: "Sheet1", address: "J4", value: "" }, + { sheet: "Sheet1", address: "B15", value: "A" }, + { sheet: "Sheet1", address: "B16", value: "B" }, + { sheet: "Sheet1", address: "H15", value: 999 }, + { sheet: "Sheet1", address: "H16", value: 0.3 }, + { sheet: "Sheet1", address: "J15", value: "" }, + { sheet: "Sheet1", address: "J16", value: "" }, + ]; + const inspection = inspectWorkbookTask({ instruction, sheetNames: ["Sheet1"], cells }); + + expect(buildWorkbookSuggestedPlan(inspection, "Sheet1").operations).toEqual([]); + expect(inspection.blockedTargets).toEqual([ + expect.objectContaining({ + sheet: "Sheet1", + address: "J16", + missingDependencies: ["I4", "J4"], + }), + ]); + const modelSuppliedPlan = verifyWorkbookPlan({ + instruction, + inspection, + cells, + sheetNames: ["Sheet1"], + operations: [ + { sheet: "Sheet1", cell: "J15", formula: 'IF(MEDIAN(H15,VLOOKUP(B15,$A$3:$J$4,{9,10},0))=H15,"Pass","Fail")' }, + { sheet: "Sheet1", cell: "J16", formula: 'IF(MEDIAN(H16,VLOOKUP(B16,$A$3:$J$4,{9,10},0))=H16,"Pass","Fail")' }, + ], + }); + expect(modelSuppliedPlan.status).toBe("needs_repair"); + expect(modelSuppliedPlan.issues).toContainEqual(expect.objectContaining({ + kind: "unsafe_lookup_bounds", + sheet: "Sheet1", + address: "J16", + })); + }); + + it("does not project sheet-qualified lookup bounds onto the target sheet", () => { + const instruction = "Users select A-G in 'Input'!B15:B17 and enter readings in 'Input'!H15:H17. Fill 'Input'!J15:J17 with Pass or Fail using the corresponding range defined in 'Rules'!I3 and 'Rules'!J3. The dropdown populates 'Input'!B3:B9."; + const cells: WorkbookObservedCell[] = [ + ...["A", "B", "C", "D", "E", "F", "G"].map((value, index) => ({ sheet: "Input", address: `A${index + 3}`, value })), + ...Array.from({ length: 7 }, (_, index) => [ + { sheet: "Rules", address: `I${index + 3}`, value: 0.2 + index / 100 }, + { sheet: "Rules", address: `J${index + 3}`, value: 0.5 + index / 100 }, + ]).flat(), + { sheet: "Input", address: "B15", value: "A" }, + { sheet: "Input", address: "B16", value: "B" }, + { sheet: "Input", address: "B17", value: "C" }, + { sheet: "Input", address: "H15", value: 0.3 }, + { sheet: "Input", address: "H16", value: 0.4 }, + { sheet: "Input", address: "H17", value: 0.5 }, + { sheet: "Input", address: "J15", value: "" }, + { sheet: "Input", address: "J16", value: "" }, + { sheet: "Input", address: "J17", value: "" }, + ]; + const inspection = inspectWorkbookTask({ instruction, sheetNames: ["Input", "Rules"], cells }); + + expect(buildWorkbookSuggestedPlan(inspection, "Input").operations).toEqual([]); + }); + + it("does not materialize unrelated formula-band repairs for a scoped task", () => { + const cells: WorkbookObservedCell[] = [ + { sheet: "ATTENDENCE", address: "F3", value: "21", formula: 'TEXT(F4,"DD")' }, + { sheet: "ATTENDENCE", address: "F4", value: "2015-10-21" }, + { sheet: "ATTENDENCE", address: "G3", value: "TH" }, + { sheet: "ATTENDENCE", address: "G4", value: "2015-10-22" }, + { sheet: "ATTENDENCE", address: "H3", value: "F" }, + { sheet: "ATTENDENCE", address: "H4", value: "2015-10-23" }, + { sheet: "ATTENDENCE", address: "B10", value: 2, formula: "A10*2" }, + { sheet: "ATTENDENCE", address: "C10", value: "" }, + { sheet: "ATTENDENCE", address: "D10", value: 8, formula: "C10*2" }, + { sheet: "ATTENDENCE", address: "E10", value: 16, formula: "D10*2" }, + ]; + const inspection = inspectWorkbookTask({ + instruction: 'Correct only the wrong weekday formula TEXT(F4,"DD") and preserve the other adjacent labels.', + sheetNames: ["ATTENDENCE"], + cells, + }); + + expect(inspection.formulaRepairSuggestions).toContainEqual(expect.objectContaining({ cell: "C10" })); + expect(buildWorkbookSuggestedPlan(inspection, "ATTENDENCE").operations).toEqual([ + { elementId: "F3", formula: 'TEXT(F4,"DDD")' }, + ]); + }); + it("rejects shifted prior-period forecast formulas and provides the visible repair references", () => { const cells: WorkbookObservedCell[] = [ { sheet: "WC_Forecast", address: "B15", value: "Cash & Equivalents" }, From e1d9842aa7259c7e2321698d9f47719dc3ca9a5c Mon Sep 17 00:00:00 2001 From: homen Date: Tue, 14 Jul 2026 23:24:19 -0700 Subject: [PATCH 04/12] fix(runtime): read unsupported workbook drawings safely Covers src runtime and tests for original-first workbook mutation reads. --- src/eval/spreadsheetBenchNodeAgentBridge.ts | 4 +- src/eval/spreadsheetBenchScorer.ts | 15 +++++ tests/spreadsheetBenchNodeAgentBridge.test.ts | 67 ++++++++++++++++++- 3 files changed, 83 insertions(+), 3 deletions(-) diff --git a/src/eval/spreadsheetBenchNodeAgentBridge.ts b/src/eval/spreadsheetBenchNodeAgentBridge.ts index 419b5b9e..0ade3fb4 100644 --- a/src/eval/spreadsheetBenchNodeAgentBridge.ts +++ b/src/eval/spreadsheetBenchNodeAgentBridge.ts @@ -3,6 +3,7 @@ import { existsSync, mkdirSync, readFileSync, realpathSync } from "node:fs"; import { basename, dirname, isAbsolute, relative, resolve } from "node:path"; import ExcelJS from "exceljs"; import type { SpreadsheetBenchTrack } from "./spreadsheetBenchAdapter"; +import { readSpreadsheetBenchWorkbookForMutation } from "./spreadsheetBenchScorer"; import { evaluateFormula, FormulaEvalError, @@ -207,8 +208,7 @@ export async function runSpreadsheetBenchNodeAgentBridge( const candidateWorkbookPath = resolve(options.candidateWorkbookPath); assertCandidateDoesNotOverwriteAgentInput(candidateWorkbookPath, task); - const workbook = new ExcelJS.Workbook(); - await workbook.xlsx.readFile(task.sourceWorkbookPath); + const workbook = await readSpreadsheetBenchWorkbookForMutation(task.sourceWorkbookPath); if (!workbook.worksheets.length) throw new Error(`SpreadsheetBench input workbook has no worksheets: ${task.sourceWorkbookPath}`); const intelligenceInstruction = [ task.manifest.instruction, diff --git a/src/eval/spreadsheetBenchScorer.ts b/src/eval/spreadsheetBenchScorer.ts index 4ade7806..8f87d7f0 100644 --- a/src/eval/spreadsheetBenchScorer.ts +++ b/src/eval/spreadsheetBenchScorer.ts @@ -518,6 +518,21 @@ export async function readSpreadsheetBenchWorkbookForCells(path: string): Promis } } +/** + * Preserve package parts that ExcelJS can read, but keep benchmark execution + * moving when a workbook contains an unsupported WPS/Office drawing package. + */ +export async function readSpreadsheetBenchWorkbookForMutation(path: string): Promise { + const workbook = new ExcelJS.Workbook(); + try { + await workbook.xlsx.readFile(path); + return workbook; + } catch (error) { + if (!isExcelJsUnsupportedPackagePartError(error)) throw error; + return readSanitizedWorkbook(path); + } +} + async function readSanitizedWorkbook(path: string): Promise { const sanitized = await sanitizeWorkbookPackageForCellRead(path); try { diff --git a/tests/spreadsheetBenchNodeAgentBridge.test.ts b/tests/spreadsheetBenchNodeAgentBridge.test.ts index ef967de6..253333dc 100644 --- a/tests/spreadsheetBenchNodeAgentBridge.test.ts +++ b/tests/spreadsheetBenchNodeAgentBridge.test.ts @@ -1,7 +1,8 @@ -import { existsSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import ExcelJS from "exceljs"; +import JSZip from "jszip"; import { afterEach, describe, expect, it } from "vitest"; import { runSpreadsheetBenchNodeAgentBridge, @@ -130,6 +131,34 @@ describe("SpreadsheetBench canonical NodeAgent bridge", () => { }); }); + it("falls back to a sanitized workbook only when an unsupported WPS drawing blocks mutation", async () => { + const root = tempRoot(); + const instruction = "Inspect the workbook and report what remains."; + const task = await stagedTask(root, instruction); + const input = join(root, "tasks", "bridge-01", "agent", "inputs", "input.xlsx"); + await addUnsupportedWpsDrawing(input); + + const directRead = new ExcelJS.Workbook(); + await expect(directRead.xlsx.readFile(input)).rejects.toThrow(/reading 'anchors'/); + + const candidate = join(root, "output", "wps-drawing.xlsx"); + const receipt = await runSpreadsheetBenchNodeAgentBridge({ + agentManifestPath: task.agentManifest, + candidateWorkbookPath: candidate, + model: invalidArtifactInspectionModel(), + traceId: "trace_sbench_wps_drawing_fallback", + maxSteps: 4, + now: () => 1_000, + }); + + expect(receipt.stages.inspect.status).toBe("completed"); + expect(receipt.frame.agentResult.stopReason).toBe("done"); + expect(existsSync(candidate)).toBe(true); + const emitted = new ExcelJS.Workbook(); + await emitted.xlsx.readFile(candidate); + expect(emitted.getWorksheet("Model")?.getCell("A1").value).toBe(2); + }); + it("keeps a syntactically verified write open when deterministic recalculation is unsupported", async () => { const root = tempRoot(); const instruction = 'Replace Model!B1 with formula TEXT(A1,"0.00"), then verify the changed cell.'; @@ -995,6 +1024,42 @@ async function stagedWorkbookWideAverageTask(root: string): Promise<{ agentManif return { agentManifest: join(agentDir, "task.json") }; } +async function addUnsupportedWpsDrawing(path: string): Promise { + const zip = await JSZip.loadAsync(readFileSync(path)); + const worksheet = zip.file("xl/worksheets/sheet1.xml"); + const contentTypes = zip.file("[Content_Types].xml"); + if (!worksheet || !contentTypes) throw new Error("fixture workbook package is incomplete"); + + zip.file( + "xl/worksheets/sheet1.xml", + (await worksheet.async("string")).replace("", ''), + ); + zip.file( + "xl/worksheets/_rels/sheet1.xml.rels", + '', + ); + zip.file( + "xl/drawings/drawing1.xml", + '10103040', + ); + zip.file( + "xl/drawings/_rels/drawing1.xml.rels", + '', + ); + zip.file( + "xl/media/image1.png", + Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=", "base64"), + ); + zip.file( + "[Content_Types].xml", + (await contentTypes.async("string")).replace( + "", + '', + ), + ); + writeFileSync(path, await zip.generateAsync({ type: "nodebuffer" })); +} + function tempRoot(): string { const root = join(tmpdir(), `noderoom-spreadsheetbench-nodeagent-${process.pid}-${Date.now()}-${roots.length}`); mkdirSync(root, { recursive: true }); From f1f3e59114fe17a5168fe09d388e6650337eee5f Mon Sep 17 00:00:00 2001 From: homen Date: Wed, 15 Jul 2026 02:17:58 -0700 Subject: [PATCH 05/12] feat(nodeagent): add provider-neutral resilient route --- src/nodeagent/integrations/modelInterop.ts | 16 +- src/nodeagent/models/adapter.ts | 280 +++++++++++++++++- src/nodeagent/models/modelCatalog.ts | 13 + .../modelAdapterProviderNeutralRoute.test.ts | 214 +++++++++++++ 4 files changed, 519 insertions(+), 4 deletions(-) create mode 100644 tests/modelAdapterProviderNeutralRoute.test.ts diff --git a/src/nodeagent/integrations/modelInterop.ts b/src/nodeagent/integrations/modelInterop.ts index 4ef09c7f..4a1ab15b 100644 --- a/src/nodeagent/integrations/modelInterop.ts +++ b/src/nodeagent/integrations/modelInterop.ts @@ -1,4 +1,4 @@ -import { resolveModelAlias } from "../models/modelCatalog"; +import { isNodeAgentFreeAutoModel, resolveModelAlias } from "../models/modelCatalog"; export type InteropRouteRuntime = "native" | "langchain" | "litellm" | "openrouter"; @@ -36,6 +36,18 @@ export function normalizeInteropModelRoute(route: string): InteropModelRoute { }; } + const resolvedNativeRoute = resolveModelAlias(requested); + if (isNodeAgentFreeAutoModel(resolvedNativeRoute)) { + return { + requested, + modelId: resolvedNativeRoute, + provider: "nodeagent", + runtime: "native", + routePolicy: "proxy", + basis: ["runtime:native", "provider:nodeagent", "policy:provider_neutral_free_first", `model:${resolvedNativeRoute}`], + }; + } + const langchain = requested.match(/^langchain:([^:]+):(.+)$/i); if (langchain) { const provider = normalizeProvider(langchain[1]); @@ -77,7 +89,7 @@ export function normalizeInteropModelRoute(route: string): InteropModelRoute { }; } - const modelId = resolveModelAlias(requested); + const modelId = resolvedNativeRoute; return { requested, modelId, diff --git a/src/nodeagent/models/adapter.ts b/src/nodeagent/models/adapter.ts index 1970ddf6..57dfac6d 100644 --- a/src/nodeagent/models/adapter.ts +++ b/src/nodeagent/models/adapter.ts @@ -20,8 +20,15 @@ import { anthropic } from "@ai-sdk/anthropic"; import { google } from "@ai-sdk/google"; import { openai, createOpenAI } from "@ai-sdk/openai"; import type { AgentModel, AgentMessage, AgentTool, AgentToolChoice, ToolCall } from "../core/types"; -import { getProviderForModel, getModelPricing, resolveModelAlias } from "./modelCatalog"; import { + getProviderForModel, + getModelPricing, + isNodeAgentFreeAutoModel, + NODEAGENT_FREE_AUTO_MODEL, + resolveModelAlias, +} from "./modelCatalog"; +import { + OPENROUTER_FREE_AUTO_MODEL, isOpenRouterFreeAutoModel, openRouterFreeCandidateSignal, openRouterFreeCandidateTimeoutMs, @@ -36,7 +43,15 @@ import { } from "./openRouterFreeModels"; import { QualityFailoverError, assessAgentToolTurnQuality, runQualityFailover, type QualityFailoverReceipt } from "./qualityFailover"; import { redactPII } from "../guardrails/gateway"; -import { assertProviderEgressAllowed, assertProviderRouteAllowed, isProviderNonRetryableError, type ProviderEgressArtifact, type ProviderRouteEntrypoint, type ProviderRouteReceipt } from "../guardrails/egressPolicy"; +import { + assertProviderEgressAllowed, + assertProviderRouteAllowed, + isProviderNonRetryableError, + providerNonRetryableReason, + type ProviderEgressArtifact, + type ProviderRouteEntrypoint, + type ProviderRouteReceipt, +} from "../guardrails/egressPolicy"; // OpenRouter = OpenAI-compatible endpoint; this is how the cheap/free models are reached. // Built lazily (per call) so process.env.OPENROUTER_API_KEY is read AFTER .env.local loads — @@ -133,6 +148,37 @@ export const priceRun = (modelId: string, inTok: number, outTok: number): number type SdkToolSet = Record; type GenerateTextResultAny = any; +const NODEAGENT_PROVIDER_NEUTRAL_POLICY = "nodeagent_provider_neutral_free_first_v1" as const; +const DEFAULT_NODEAGENT_GOOGLE_FALLBACK_MODEL = "gemini-3-flash-preview"; + +type ProviderNeutralBilling = { + /** This describes route pricing, not the account's eventual invoice or free-tier entitlement. */ + free: boolean; + classification: "openrouter_zero_price_route" | "catalog_priced" | "pricing_unverified"; + inputPer1M?: number; + outputPer1M?: number; + basis: string[]; +}; + +type ProviderNeutralRouteReceipt = ProviderRouteReceipt & { + providerNeutral: { + policy: typeof NODEAGENT_PROVIDER_NEUTRAL_POLICY; + requestedModel: typeof NODEAGENT_FREE_AUTO_MODEL; + primary: { + route: typeof OPENROUTER_FREE_AUTO_MODEL; + outcome: "accepted" | "provider_wide_exhausted"; + reason?: string; + qualityFailover?: QualityFailoverReceipt; + }; + selected: { + route: "openrouter_free" | "google_direct"; + model: string; + provider: "openrouter" | "gemini"; + billing: ProviderNeutralBilling; + }; + }; +}; + // ── Production reliability: retry transient failures (429/5xx/network) with exp backoff + jitter, // honoring the deadline AbortSignal, plus an optional cross-model fallback. (async_reliability layer 2) const TRANSIENT_RE = /(\b429\b|\b5\d\d\b|rate.?limit|overloaded|temporar|timed?.?out|ECONNRESET|ETIMEDOUT|EAI_AGAIN|ENOTFOUND|fetch failed|socket hang up|service unavailable)/i; @@ -188,6 +234,19 @@ async function generateAgentText( providerRoute?: ProviderRouteReceipt; qualityFailover?: QualityFailoverReceipt; }> { + if (isNodeAgentFreeAutoModel(modelId)) { + return generateProviderNeutralAgentText({ + system, + messages, + sdkTools, + signal, + entrypoint, + artifacts, + toolChoice, + freeAutoMode, + qualityContext, + }); + } if (!isOpenRouterFreeAutoModel(modelId)) { const call = async (id: string) => { const providerRoute = assertProviderRouteAllowed({ model: id, entrypoint, env: process.env }); @@ -278,6 +337,223 @@ async function generateAgentText( throw adapterQualityFailoverError(routed.receipt, routed.lastError); } +async function generateProviderNeutralAgentText(args: { + system: string; + messages: ModelMessage[]; + sdkTools: SdkToolSet; + signal?: AbortSignal; + entrypoint: ProviderRouteEntrypoint; + artifacts: ProviderEgressArtifact[]; + toolChoice?: AgentToolChoice; + freeAutoMode?: OpenRouterFreeModelMode; + qualityContext?: { messages: AgentMessage[]; tools: AgentTool[] }; +}): Promise<{ + res: GenerateTextResultAny; + resolvedModel: string; + providerRoute: ProviderRouteReceipt; + qualityFailover?: QualityFailoverReceipt; +}> { + try { + const primary = await generateAgentText( + OPENROUTER_FREE_AUTO_MODEL, + args.system, + args.messages, + args.sdkTools, + args.signal, + args.entrypoint, + args.artifacts, + args.toolChoice, + args.freeAutoMode, + args.qualityContext, + ); + if (!primary.providerRoute) { + throw new Error(`${NODEAGENT_FREE_AUTO_MODEL} primary route returned no provider receipt`); + } + return { + ...primary, + providerRoute: providerNeutralRouteReceipt({ + selectedReceipt: primary.providerRoute, + resolvedModel: primary.resolvedModel, + primaryOutcome: "accepted", + }), + }; + } catch (primaryError) { + const exhaustion = openRouterProviderWideExhaustion(primaryError); + if (!exhaustion || args.signal?.aborted) throw primaryError; + + const configuredKey = envValue("GOOGLE_GENERATIVE_AI_API_KEY"); + if (!configuredKey) { + throw providerNeutralRecoveryError( + primaryError, + `${exhaustion.reason}; direct Google fallback is unavailable because GOOGLE_GENERATIVE_AI_API_KEY is not configured`, + ); + } + + const fallbackModel = resolveModelAlias( + envValue("NODEAGENT_FREE_AUTO_GOOGLE_MODEL") ?? DEFAULT_NODEAGENT_GOOGLE_FALLBACK_MODEL, + ); + if (getProviderForModel(fallbackModel) !== "gemini") { + throw providerNeutralRecoveryError( + primaryError, + `${exhaustion.reason}; NODEAGENT_FREE_AUTO_GOOGLE_MODEL must resolve to a direct Gemini model`, + ); + } + + let providerRoute: ProviderRouteReceipt; + try { + providerRoute = assertProviderRouteAllowed({ model: fallbackModel, entrypoint: args.entrypoint, env: process.env }); + assertProviderEgressAllowed({ model: fallbackModel, entrypoint: args.entrypoint, artifacts: args.artifacts, env: process.env }); + } catch (policyError) { + throw providerNeutralRecoveryError( + primaryError, + `${exhaustion.reason}; direct Google fallback blocked before provider call: ${shortProviderError(policyError)}`, + policyError, + ); + } + + try { + const res = await withRetry(() => generateText({ + model: google(fallbackModel), + system: args.system, + messages: args.messages, + tools: args.sdkTools, + toolChoice: Object.keys(args.sdkTools).length ? sdkToolChoiceForModel(fallbackModel, args.toolChoice) : undefined, + abortSignal: args.signal, + }), args.signal); + return { + res, + resolvedModel: fallbackModel, + providerRoute: providerNeutralRouteReceipt({ + selectedReceipt: providerRoute, + resolvedModel: fallbackModel, + primaryOutcome: "provider_wide_exhausted", + primaryReason: exhaustion.reason, + primaryQualityFailover: exhaustion.receipt, + }), + }; + } catch (fallbackError) { + throw providerNeutralRecoveryError( + primaryError, + `${exhaustion.reason}; direct Google fallback failed: ${shortProviderError(fallbackError)}`, + fallbackError, + ); + } + } +} + +function openRouterProviderWideExhaustion(error: unknown): { + reason: string; + receipt?: QualityFailoverReceipt; +} | undefined { + if (error instanceof QualityFailoverError) { + const terminal = error.receipt.terminalFailure; + if (terminal?.failureClass === "provider" + && terminal.providerFailureScope === "global" + && terminal.providerFailureCategory === "quota") { + return { reason: terminal.reason, receipt: error.receipt }; + } + return undefined; + } + const reason = providerNonRetryableReason(error); + return reason && /quota|credit/i.test(reason) ? { reason } : undefined; +} + +function providerNeutralRecoveryError( + primaryError: unknown, + detail: string, + cause: unknown = primaryError, +): Error { + const message = `${NODEAGENT_FREE_AUTO_MODEL} could not recover after OpenRouter provider-wide exhaustion: ${detail}`; + if (primaryError instanceof QualityFailoverError) { + return new QualityFailoverError(message, primaryError.receipt, cause, primaryError.usage); + } + return new Error(message, { cause }); +} + +function providerNeutralRouteReceipt(args: { + selectedReceipt: ProviderRouteReceipt; + resolvedModel: string; + primaryOutcome: "accepted" | "provider_wide_exhausted"; + primaryReason?: string; + primaryQualityFailover?: QualityFailoverReceipt; +}): ProviderNeutralRouteReceipt { + const provider = args.selectedReceipt.provider; + if (provider !== "openrouter" && provider !== "gemini") { + throw new Error(`${NODEAGENT_FREE_AUTO_MODEL} selected unsupported provider ${provider}`); + } + const selectedRoute = provider === "openrouter" ? "openrouter_free" : "google_direct"; + const billing = providerNeutralBilling(args.resolvedModel, provider); + return { + ...args.selectedReceipt, + requestedModel: NODEAGENT_FREE_AUTO_MODEL, + resolvedModel: args.resolvedModel, + basis: [ + `requested:${NODEAGENT_FREE_AUTO_MODEL}`, + `resolved:${args.resolvedModel}`, + `provider_neutral_policy:${NODEAGENT_PROVIDER_NEUTRAL_POLICY}`, + `primary_route:${OPENROUTER_FREE_AUTO_MODEL}`, + `primary_outcome:${args.primaryOutcome}`, + ...(args.primaryReason ? [`primary_reason:${args.primaryReason}`] : []), + ...args.selectedReceipt.basis.map((basis) => `selected_route:${basis}`), + ...billing.basis.map((basis) => `billing:${basis}`), + ], + providerNeutral: { + policy: NODEAGENT_PROVIDER_NEUTRAL_POLICY, + requestedModel: NODEAGENT_FREE_AUTO_MODEL, + primary: { + route: OPENROUTER_FREE_AUTO_MODEL, + outcome: args.primaryOutcome, + ...(args.primaryReason ? { reason: args.primaryReason } : {}), + ...(args.primaryQualityFailover ? { qualityFailover: args.primaryQualityFailover } : {}), + }, + selected: { + route: selectedRoute, + model: args.resolvedModel, + provider, + billing, + }, + }, + }; +} + +function providerNeutralBilling( + resolvedModel: string, + provider: "openrouter" | "gemini", +): ProviderNeutralBilling { + const pricing = getModelPricing(resolvedModel); + if (provider === "openrouter" + && resolvedModel.toLowerCase().endsWith(":free") + && pricing?.inputPer1M === 0 + && pricing.outputPer1M === 0) { + return { + free: true, + classification: "openrouter_zero_price_route", + inputPer1M: 0, + outputPer1M: 0, + basis: ["provider:openrouter", "model_suffix::free", "catalog_input_per_1m:0", "catalog_output_per_1m:0"], + }; + } + if (pricing) { + return { + free: false, + classification: "catalog_priced", + inputPer1M: pricing.inputPer1M, + outputPer1M: pricing.outputPer1M, + basis: [ + `provider:${provider}`, + `catalog_input_per_1m:${pricing.inputPer1M}`, + `catalog_output_per_1m:${pricing.outputPer1M}`, + "account_free_tier:not_asserted", + ], + }; + } + return { + free: false, + classification: "pricing_unverified", + basis: [`provider:${provider}`, "account_free_tier:not_asserted", "catalog_pricing:missing"], + }; +} + function qualityAttemptedRoutes(receipt: QualityFailoverReceipt): string[] { return receipt.routeAttempts.map((attempt) => attempt.routeId); } diff --git a/src/nodeagent/models/modelCatalog.ts b/src/nodeagent/models/modelCatalog.ts index ee357043..b55a780e 100644 --- a/src/nodeagent/models/modelCatalog.ts +++ b/src/nodeagent/models/modelCatalog.ts @@ -35,6 +35,13 @@ import { OPENROUTER_FREE_AUTO_MODEL, OPENROUTER_FREE_META_MODEL, freeOpenRouterPricing } from "./openRouterFreeModels"; +/** Provider-neutral, free-first NodeAgent route. Concrete providers are resolved by the adapter. */ +export const NODEAGENT_FREE_AUTO_MODEL = "nodeagent/free-auto"; + +export function isNodeAgentFreeAutoModel(modelId: string): boolean { + return modelId.trim().toLowerCase() === NODEAGENT_FREE_AUTO_MODEL; +} + export type LlmProvider = "openai" | "anthropic" | "gemini" | "openrouter" | "xai" | "nebius"; export type LlmTask = @@ -426,6 +433,9 @@ export function getModelPricing(modelName: string): ModelPricing | null { */ export function getProviderForModel(modelName: string): LlmProvider | null { const resolved = modelAliases[modelName.toLowerCase().trim()] ?? modelName; + if (isNodeAgentFreeAutoModel(resolved)) { + return null; + } if (resolved.startsWith("nebius/")) { return "nebius"; } @@ -632,6 +642,9 @@ export const modelAliases: Record = { "openrouter-free-auto": OPENROUTER_FREE_AUTO_MODEL, "openrouter/free-auto": OPENROUTER_FREE_AUTO_MODEL, "openrouter/free": OPENROUTER_FREE_META_MODEL, + "nodeagent-auto": NODEAGENT_FREE_AUTO_MODEL, + "nodeagent-free-auto": NODEAGENT_FREE_AUTO_MODEL, + "nodeagent/free-auto": NODEAGENT_FREE_AUTO_MODEL, "kimi": "moonshotai/kimi-k2.7-code", "kimi-k2": "moonshotai/kimi-k2.7-code", "kimi-k2.6": "moonshotai/kimi-k2.6", diff --git a/tests/modelAdapterProviderNeutralRoute.test.ts b/tests/modelAdapterProviderNeutralRoute.test.ts new file mode 100644 index 00000000..13556d13 --- /dev/null +++ b/tests/modelAdapterProviderNeutralRoute.test.ts @@ -0,0 +1,214 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { normalizeInteropModelRoute } from "../src/nodeagent/integrations/modelInterop"; +import { + getProviderForModel, + NODEAGENT_FREE_AUTO_MODEL, + resolveModelAlias, +} from "../src/nodeagent/models/modelCatalog"; +import { resetOpenRouterFreeRouteHealth } from "../src/nodeagent/models/openRouterFreeModels"; +import { QualityFailoverError } from "../src/nodeagent/models/qualityFailover"; + +const generateTextMock = vi.hoisted(() => vi.fn()); + +vi.mock("ai", () => ({ + generateText: generateTextMock, + tool: (definition: unknown) => definition, +})); + +vi.mock("@ai-sdk/openai", () => ({ + createOpenAI: () => ({ + chat: (id: string) => ({ provider: "openrouter", id }), + }), + openai: (id: string) => ({ provider: "openai", id }), +})); + +vi.mock("@ai-sdk/anthropic", () => ({ + anthropic: (id: string) => ({ provider: "anthropic", id }), +})); + +vi.mock("@ai-sdk/google", () => ({ + google: (id: string) => ({ provider: "google", id }), +})); + +const OPENROUTER_MODEL = "nvidia/nemotron-3-super-120b-a12b:free"; +const GOOGLE_MODEL = "gemini-3-flash-preview"; +const ROUTING_ENV = [ + "OPENROUTER_API_KEY", + "OPENROUTER_FREE_MODEL_CACHE_MS", + "OPENROUTER_FREE_AUTO_LIMIT", + "OPENROUTER_FREE_CANDIDATE_RETRIES", + "OPENROUTER_FREE_CANDIDATE_TIMEOUT_MS", + "OPENROUTER_FREE_REQUEST_TIMEOUT_MS", + "OPENROUTER_FREE_REQUEST_RESERVE_MS", + "GOOGLE_GENERATIVE_AI_API_KEY", + "NODEAGENT_FREE_AUTO_GOOGLE_MODEL", + "NODEAGENT_ALLOWED_PROVIDERS", + "PROVIDER_EGRESS_ALLOWED_PROVIDERS", + "PROVIDER_EGRESS_REQUIRE_ALLOWLIST", + "NODEROOM_PRODUCTION", +] as const; + +function clearRoutingEnv() { + for (const name of ROUTING_ENV) delete process.env[name]; +} + +function successfulTurn(modelId: string) { + return { + text: JSON.stringify({ selectedModel: modelId }), + toolCalls: [] as Array<{ toolCallId: string; toolName: string; input: Record }>, + usage: { inputTokens: 10, outputTokens: 5 }, + }; +} + +async function runNeutralRoute() { + const { model } = await import("../src/nodeagent/models/adapter"); + const route = model(NODEAGENT_FREE_AUTO_MODEL, { freeAutoMode: "structured", entrypoint: "system" }); + const step = await route.next({ + system: "Return JSON only.", + messages: [{ role: "user", content: "emit a structured edit plan" }], + tools: [], + }); + return { route, step }; +} + +describe("provider-neutral NodeAgent free-first routing", () => { + beforeEach(() => { + clearRoutingEnv(); + generateTextMock.mockReset(); + generateTextMock.mockImplementation(async (options: { model: { id: string } }) => successfulTurn(options.model.id)); + process.env.OPENROUTER_API_KEY = "test-openrouter-key"; + process.env.OPENROUTER_FREE_MODEL_CACHE_MS = "0"; + process.env.OPENROUTER_FREE_AUTO_LIMIT = "1"; + process.env.OPENROUTER_FREE_CANDIDATE_RETRIES = "0"; + process.env.OPENROUTER_FREE_CANDIDATE_TIMEOUT_MS = "5000"; + process.env.OPENROUTER_FREE_REQUEST_TIMEOUT_MS = "90000"; + process.env.OPENROUTER_FREE_REQUEST_RESERVE_MS = "5000"; + resetOpenRouterFreeRouteHealth(); + vi.stubGlobal("fetch", async () => new Response(JSON.stringify({ + data: [{ + id: OPENROUTER_MODEL, + pricing: { prompt: "0", completion: "0", request: "0" }, + context_length: 1_000_000, + supported_parameters: ["max_tokens", "response_format", "structured_outputs", "tool_choice", "tools"], + }], + }), { status: 200 })); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + clearRoutingEnv(); + resetOpenRouterFreeRouteHealth(); + }); + + it("uses the governed OpenRouter free ladder first and preserves concrete identity", async () => { + const { route, step } = await runNeutralRoute(); + + expect(generateTextMock.mock.calls.map((call) => call[0].model)).toEqual([ + { provider: "openrouter", id: OPENROUTER_MODEL }, + ]); + expect(route.name).toBe(OPENROUTER_MODEL); + expect(step.providerRoute).toMatchObject({ + requestedModel: NODEAGENT_FREE_AUTO_MODEL, + resolvedModel: OPENROUTER_MODEL, + provider: "openrouter", + providerNeutral: { + policy: "nodeagent_provider_neutral_free_first_v1", + primary: { route: "openrouter/free-auto", outcome: "accepted" }, + selected: { + route: "openrouter_free", + model: OPENROUTER_MODEL, + provider: "openrouter", + billing: { free: true, classification: "openrouter_zero_price_route" }, + }, + }, + qualityFailover: { status: "succeeded", selectedRouteId: OPENROUTER_MODEL }, + }); + }); + + it("falls back to direct Gemini only after provider-wide OpenRouter quota exhaustion", async () => { + process.env.GOOGLE_GENERATIVE_AI_API_KEY = "test-google-key"; + generateTextMock + .mockRejectedValueOnce(new Error("Provider request failed 429: free-models-per-day-high-balance rate limit exceeded")) + .mockImplementationOnce(async (options: { model: { id: string } }) => successfulTurn(options.model.id)); + + const { route, step } = await runNeutralRoute(); + + expect(generateTextMock.mock.calls.map((call) => call[0].model)).toEqual([ + { provider: "openrouter", id: OPENROUTER_MODEL }, + { provider: "google", id: GOOGLE_MODEL }, + ]); + expect(route.name).toBe(GOOGLE_MODEL); + expect(step.providerRoute).toMatchObject({ + requestedModel: NODEAGENT_FREE_AUTO_MODEL, + resolvedModel: GOOGLE_MODEL, + provider: "gemini", + providerNeutral: { + primary: { + route: "openrouter/free-auto", + outcome: "provider_wide_exhausted", + reason: "provider_free_quota_exhausted", + qualityFailover: { + stopReason: "global_provider_failure", + terminalFailure: { providerFailureScope: "global", providerFailureCategory: "quota" }, + }, + }, + selected: { + route: "google_direct", + model: GOOGLE_MODEL, + provider: "gemini", + billing: { + free: false, + classification: "catalog_priced", + inputPer1M: 0.5, + outputPer1M: 3, + }, + }, + }, + }); + expect(step.providerRoute?.basis).toContain("billing:account_free_tier:not_asserted"); + }); + + it("does not cross providers for an ordinary candidate-scoped OpenRouter failure", async () => { + process.env.GOOGLE_GENERATIVE_AI_API_KEY = "test-google-key"; + generateTextMock.mockRejectedValueOnce(new Error("Provider request failed 503: route unavailable")); + + const error = await runNeutralRoute().then(() => undefined, (failure: unknown) => failure); + + expect(error).toBeInstanceOf(QualityFailoverError); + expect(generateTextMock).toHaveBeenCalledTimes(1); + expect(generateTextMock.mock.calls[0]?.[0].model).toEqual({ provider: "openrouter", id: OPENROUTER_MODEL }); + }); + + it("does not call Google when its credential is missing", async () => { + generateTextMock.mockRejectedValueOnce(new Error("Provider request failed 429: free-models-per-day-high-balance")); + + const error = await runNeutralRoute().then(() => undefined, (failure: unknown) => failure); + + expect(error).toBeInstanceOf(QualityFailoverError); + expect(String((error as Error).message)).toContain("GOOGLE_GENERATIVE_AI_API_KEY is not configured"); + expect(generateTextMock).toHaveBeenCalledTimes(1); + }); + + it("checks provider policy before calling the configured Google fallback", async () => { + process.env.GOOGLE_GENERATIVE_AI_API_KEY = "test-google-key"; + process.env.NODEAGENT_ALLOWED_PROVIDERS = "openrouter"; + generateTextMock.mockRejectedValueOnce(new Error("Provider request failed 429: free-models-per-day-high-balance")); + + const error = await runNeutralRoute().then(() => undefined, (failure: unknown) => failure); + + expect(error).toBeInstanceOf(QualityFailoverError); + expect(String((error as Error).message)).toContain("provider_route_blocked:provider_not_allowed"); + expect(generateTextMock).toHaveBeenCalledTimes(1); + }); + + it("normalizes the logical alias without mislabeling it as OpenRouter", () => { + expect(resolveModelAlias("nodeagent-free-auto")).toBe(NODEAGENT_FREE_AUTO_MODEL); + expect(getProviderForModel(NODEAGENT_FREE_AUTO_MODEL)).toBeNull(); + expect(normalizeInteropModelRoute("nodeagent-free-auto")).toMatchObject({ + modelId: NODEAGENT_FREE_AUTO_MODEL, + provider: "nodeagent", + runtime: "native", + routePolicy: "proxy", + }); + }); +}); From 24d2e6c3ab5021851ddc6cf23a7443a02c98f4df Mon Sep 17 00:00:00 2001 From: homen Date: Wed, 15 Jul 2026 03:01:33 -0700 Subject: [PATCH 06/12] fix(nodeagent): harden workbook proof repair --- package.json | 1 + .../spreadsheetbench-merge-repair-report.ts | 265 ++++++++++++++++++ src/eval/spreadsheetBenchNodeAgentBridge.ts | 135 ++++++++- src/eval/spreadsheetBenchReportRepair.ts | 253 +++++++++++++++++ src/nodeagent/models/qualityFailover.ts | 2 +- src/nodeagent/traces/traceRedaction.ts | 40 ++- .../modelAdapterProviderNeutralRoute.test.ts | 4 +- tests/nodeagentTraceSpine.test.ts | 19 ++ tests/qualityFailover.test.ts | 6 + tests/spreadsheetBenchNodeAgentBridge.test.ts | 93 ++++++ tests/spreadsheetBenchReportRepair.test.ts | 213 ++++++++++++++ 11 files changed, 1005 insertions(+), 26 deletions(-) create mode 100644 scripts/spreadsheetbench-merge-repair-report.ts create mode 100644 src/eval/spreadsheetBenchReportRepair.ts create mode 100644 tests/spreadsheetBenchReportRepair.test.ts diff --git a/package.json b/package.json index ddf12dc1..5f70b07c 100644 --- a/package.json +++ b/package.json @@ -58,6 +58,7 @@ "benchmark:spreadsheetbench:stage": "tsx scripts/spreadsheetbench-stage.ts", "benchmark:spreadsheetbench:run": "tsx scripts/spreadsheetbench-run.ts", "benchmark:spreadsheetbench:run-chunked": "tsx scripts/spreadsheetbench-run-chunked.ts", + "benchmark:spreadsheetbench:merge-repair": "tsx scripts/spreadsheetbench-merge-repair-report.ts", "benchmark:spreadsheetbench:rescore-existing": "tsx scripts/spreadsheetbench-rescore-existing.ts", "benchmark:spreadsheetbench:official-v1-project": "tsx scripts/spreadsheetbench-official-v1-project.ts", "benchmark:spreadsheetbench:official-v1-project-chunked": "tsx scripts/spreadsheetbench-official-v1-project-chunked.ts", diff --git a/scripts/spreadsheetbench-merge-repair-report.ts b/scripts/spreadsheetbench-merge-repair-report.ts new file mode 100644 index 00000000..96d7edd6 --- /dev/null +++ b/scripts/spreadsheetbench-merge-repair-report.ts @@ -0,0 +1,265 @@ +import { createHash } from "node:crypto"; +import { + existsSync, + mkdirSync, + readFileSync, + statSync, + writeFileSync, +} from "node:fs"; +import { basename, dirname, isAbsolute, relative, resolve } from "node:path"; +import { + assertSpreadsheetBenchMergeOutputPaths, + mergeSpreadsheetBenchRepairReport, +} from "../src/eval/spreadsheetBenchReportRepair"; +import type { + SpreadsheetBenchSidecarFileEvidence, + SpreadsheetBenchRunnerReport, + SpreadsheetBenchRunnerTaskResult, +} from "../src/eval/spreadsheetBenchRunner"; + +const args = process.argv.slice(2); +const baseReportPath = requiredOption("--base-report"); +const baseRunRoot = requiredOption("--base-run-root"); +const repairReportPath = requiredOption("--repair-report"); +const repairRunRoot = requiredOption("--repair-run-root"); +const replacementTaskIdsPath = requiredOption("--task-ids-file"); +const outputRunRoot = requiredOption("--output-run-root"); +const jsonOut = requiredOption("--json-out"); +const receiptOut = requiredOption("--receipt-out"); + +const baseReportAbsolute = resolve(baseReportPath); +const repairReportAbsolute = resolve(repairReportPath); +const replacementTaskIdsAbsolute = resolve(replacementTaskIdsPath); +const baseRoot = resolve(baseRunRoot); +const repairRoot = resolve(repairRunRoot); +const outputRoot = resolve(outputRunRoot); +const reportOutput = resolve(jsonOut); +const receiptOutput = resolve(receiptOut); +assertInPlaceRunRoot(outputRoot, baseRoot, repairRoot); + +const baseContent = readFileSync(baseReportAbsolute); +const repairContent = readFileSync(repairReportAbsolute); +const base = JSON.parse(baseContent.toString("utf8")) as SpreadsheetBenchRunnerReport; +const repair = JSON.parse(repairContent.toString("utf8")) as SpreadsheetBenchRunnerReport; +const replacementTaskIds = readTaskIds(replacementTaskIdsAbsolute); +assertRunRoot(baseRoot, base.outputRoot, "base"); +assertRunRoot(repairRoot, repair.outputRoot, "repair"); +assertRunRoot(outputRoot, basename(outputRoot), "output"); + +const generatedAt = new Date().toISOString(); +const baseReportSha256 = sha256(baseContent); +const repairReportSha256 = sha256(repairContent); +const merged = mergeSpreadsheetBenchRepairReport({ + base, + repair, + replacementTaskIds, + generatedAt, + outputRoot: basename(outputRoot), + baseReportSha256, + repairReportSha256, +}); + +const repairIds = new Set(replacementTaskIds); +const verifiedArtifactDirectories: Array<{ taskId: string; source: "base" | "repair"; directory: string }> = []; +const seenArtifactDirectories = new Set(); +const protectedArtifactPaths: string[] = []; +for (const result of merged.results) { + const source = repairIds.has(result.taskId) ? "repair" : "base"; + const verified = verifyResultArtifacts(outputRoot, result); + const artifactDirectory = verified.directory; + if (seenArtifactDirectories.has(artifactDirectory)) { + throw new Error(`multiple tasks resolve to the same artifact directory: ${artifactDirectory}`); + } + seenArtifactDirectories.add(artifactDirectory); + protectedArtifactPaths.push(...verified.files); + verifiedArtifactDirectories.push({ taskId: result.taskId, source, directory: artifactDirectory }); +} +assertSpreadsheetBenchMergeOutputPaths({ + reportOutput, + receiptOutput, + protectedPaths: [baseReportAbsolute, repairReportAbsolute, replacementTaskIdsAbsolute, ...protectedArtifactPaths], +}); + +const reportContent = Buffer.from(`${JSON.stringify(merged, null, 2)}\n`); +mkdirSync(dirname(reportOutput), { recursive: true }); +writeFileSync(reportOutput, reportContent); +const reportSha256 = sha256(reportContent); +const receipt = { + schema: 1, + kind: "spreadsheetbench-nodeagent-repair-merge", + generatedAt, + base: { + report: rel(baseReportAbsolute), + reportSha256: baseReportSha256, + runRoot: rel(baseRoot), + taskCount: base.taskCount, + }, + repair: { + report: rel(repairReportAbsolute), + reportSha256: repairReportSha256, + runRoot: rel(repairRoot), + taskCount: repair.taskCount, + }, + output: { + report: rel(reportOutput), + reportSha256, + runRoot: rel(outputRoot), + taskCount: merged.taskCount, + authenticModelCalls: merged.harness.budget.modelCalls, + providerCostUsd: merged.harness.budget.providerCostUsd, + }, + replacementTaskIds, + verifiedArtifactDirectories, + invariants: { + uniqueTaskIds: new Set(merged.results.map((result) => result.taskId)).size === merged.taskCount, + toolPolicy: merged.harness.toolPolicy, + evaluatorAccess: merged.harness.evaluatorAccess, + officialProjectorUnchanged: true, + }, +}; +mkdirSync(dirname(receiptOutput), { recursive: true }); +writeFileSync(receiptOutput, `${JSON.stringify(receipt, null, 2)}\n`); + +console.log(`merged ${replacementTaskIds.length} repaired tasks into ${merged.taskCount} results`); +console.log(`wrote ${rel(reportOutput)} sha256=${reportSha256}`); +console.log(`wrote ${rel(receiptOutput)}`); + +function verifyResultArtifacts(root: string, result: SpreadsheetBenchRunnerTaskResult): { directory: string; files: string[] } { + if (!result.candidateWorkbook) throw new Error(`candidate workbook is missing: ${result.taskId}`); + const candidate = safeRelativePath(result.candidateWorkbook); + const artifactDirectory = candidate.split("/")[0]; + if (!artifactDirectory) throw new Error(`candidate artifact directory is missing: ${result.taskId}`); + const candidatePath = resolveWithin(root, candidate); + if (!existsSync(candidatePath) || !statSync(candidatePath).isFile()) { + throw new Error(`candidate workbook does not exist: ${result.taskId} ${candidate}`); + } + const files = [candidatePath]; + + for (const evidence of resultEvidence(result)) { + const path = safeRelativePath(evidence.path); + if (path.split("/")[0] !== artifactDirectory) { + throw new Error(`artifact evidence crosses task directories: ${result.taskId} ${path}`); + } + const absolute = resolveWithin(root, path); + if (!existsSync(absolute) || !statSync(absolute).isFile()) { + throw new Error(`artifact evidence does not exist: ${result.taskId} ${path}`); + } + const content = readFileSync(absolute); + if (content.length !== evidence.bytes || sha256(content) !== evidence.sha256.toLowerCase()) { + throw new Error(`artifact evidence hash mismatch: ${result.taskId} ${path}`); + } + files.push(absolute); + } + return { directory: artifactDirectory, files }; +} + +function resultEvidence(result: SpreadsheetBenchRunnerTaskResult): SpreadsheetBenchSidecarFileEvidence[] { + const sidecars = result.sidecarEvidence; + if (!sidecars) throw new Error(`sidecar evidence is missing: ${result.taskId}`); + const evidence = [ + requireFileEvidence(result.scorerReceipt, `${result.taskId} scorer receipt`), + requireFileEvidence(sidecars.candidateManifest, `${result.taskId} candidate manifest`), + requireFileEvidence(sidecars.nodeAgentReceipt, `${result.taskId} NodeAgent receipt`), + requireFileEvidence(sidecars.nodeAgentTrace, `${result.taskId} NodeAgent trace`), + ]; + const optionalFileKeys = [ + "agentWorkspaceManifest", + "editPlan", + "rawModelOutput", + "workbookInspection", + "editVerification", + ] as const; + for (const key of optionalFileKeys) { + const value = sidecars[key]; + if (value !== undefined) evidence.push(requireFileEvidence(value, `${result.taskId} ${key}`)); + } + if (sidecars.repairOutputs !== undefined) { + if (!Array.isArray(sidecars.repairOutputs)) throw new Error(`malformed repair outputs: ${result.taskId}`); + for (const [index, value] of sidecars.repairOutputs.entries()) { + evidence.push(requireFileEvidence(value, `${result.taskId} repair output ${index}`)); + } + } + return evidence; +} + +function requireFileEvidence(value: unknown, label: string): SpreadsheetBenchSidecarFileEvidence { + if (!isFileEvidence(value)) throw new Error(`malformed file evidence: ${label}`); + return value; +} + +function isFileEvidence(value: unknown): value is SpreadsheetBenchSidecarFileEvidence { + if (!value || typeof value !== "object" || Array.isArray(value)) return false; + const candidate = value as Partial; + return typeof candidate.path === "string" + && typeof candidate.sha256 === "string" + && /^[a-f0-9]{64}$/i.test(candidate.sha256) + && Number.isSafeInteger(candidate.bytes) + && (candidate.bytes ?? -1) >= 0; +} + +function readTaskIds(path: string): string[] { + const value = JSON.parse(readFileSync(resolve(path), "utf8")) as unknown; + if (!Array.isArray(value) || value.some((item) => typeof item !== "string" || !item.trim())) { + throw new Error(`--task-ids-file must contain a JSON array of non-empty strings: ${path}`); + } + const normalized = value.map((item) => (item as string).trim()); + if (new Set(normalized).size !== normalized.length) throw new Error(`--task-ids-file contains duplicates: ${path}`); + return normalized; +} + +function assertRunRoot(root: string, reportOutputRoot: string, label: string): void { + if (!existsSync(root) || !statSync(root).isDirectory()) throw new Error(`${label} run root does not exist: ${root}`); + if (basename(root).toLowerCase() !== reportOutputRoot.toLowerCase()) { + throw new Error(`${label} run root basename does not match report outputRoot: ${basename(root)} != ${reportOutputRoot}`); + } +} + +function assertInPlaceRunRoot(output: string, base: string, repair: string): void { + if (output.toLowerCase() !== base.toLowerCase() || output.toLowerCase() !== repair.toLowerCase()) { + throw new Error("base, repair, and output run roots must be the same because NodeAgent receipts bind absolute candidate paths"); + } +} + +function safeRelativePath(path: string): string { + const normalized = path.replace(/\\/g, "/"); + if (!normalized || isAbsolute(normalized) || normalized.split("/").some((part) => part === ".." || !part)) { + throw new Error(`artifact path is not a safe relative path: ${path}`); + } + return normalized; +} + +function resolveWithin(root: string, path: string): string { + const absolute = resolve(root, safeRelativePath(path)); + const relativePath = relative(root, absolute); + if (!relativePath || relativePath.startsWith("..") || isAbsolute(relativePath)) { + throw new Error(`path escapes or resolves to the run root: ${path}`); + } + return absolute; +} + +function sha256(value: Buffer): string { + return createHash("sha256").update(value).digest("hex"); +} + +function requiredOption(name: string): string { + const value = optionValue(name); + if (!value) { + console.error([ + "Usage:", + " npm run benchmark:spreadsheetbench:merge-repair -- --base-report --base-run-root --repair-report --repair-run-root --task-ids-file --output-run-root --json-out --receipt-out ", + ].join("\n")); + process.exit(2); + } + return value; +} + +function optionValue(name: string): string | undefined { + const prefixed = args.find((arg) => arg.startsWith(`${name}=`)); + if (prefixed) return prefixed.slice(name.length + 1); + const index = args.indexOf(name); + return index >= 0 ? args[index + 1] : undefined; +} + +function rel(path: string): string { + return relative(process.cwd(), path).replace(/\\/g, "/"); +} diff --git a/src/eval/spreadsheetBenchNodeAgentBridge.ts b/src/eval/spreadsheetBenchNodeAgentBridge.ts index 0ade3fb4..b621bfa4 100644 --- a/src/eval/spreadsheetBenchNodeAgentBridge.ts +++ b/src/eval/spreadsheetBenchNodeAgentBridge.ts @@ -70,10 +70,14 @@ const BRIDGE_TOOL_NAMES = [ const EXECUTE_VERIFIED_WORKBOOK_PLAN_TOOL_NAME = "execute_verified_workbook_plan"; const WRITE_TOOL_NAMES = new Set(["write_locked_cell", "write_locked_cells"]); const MUTATION_TOOL_NAMES = new Set([...WRITE_TOOL_NAMES, EXECUTE_VERIFIED_WORKBOOK_PLAN_TOOL_NAME]); -const A1_RE = /^\$?[A-Z]{1,3}\$?[1-9][0-9]*$/i; +const READ_REFERENCE_RE = /^(?:(?:'((?:[^']|'')+)'|([^!]+))!\s*)?(\$?[A-Z]{1,3}\$?[1-9][0-9]*)(?:\s*:\s*(\$?[A-Z]{1,3}\$?[1-9][0-9]*))?$/i; +const EXCEL_MAX_ROW = 1_048_576; +const EXCEL_MAX_COLUMN = 16_384; const DEFAULT_SNAPSHOT_MAX_CELLS = 1_200; const DEFAULT_SCAN_MAX_CELLS = 50_000; +class SpreadsheetBenchReadReferenceError extends Error {} + type StagedAgentManifest = { schema: 1; taskId: string; @@ -408,18 +412,53 @@ class SpreadsheetBenchWorkbookRoomTools implements RoomTools { id: sheet.name, title: sheet.name, kind: "sheet", - readHint: `Use artifactId ${JSON.stringify(sheet.name)} with A1 addresses.`, + readHint: `Use artifactId ${JSON.stringify(sheet.name)} with A1 cells or bounded ranges such as B2:F20.`, })); } async readRange(elementIds: string[], artifactId?: string): Promise { this.recalculateChangedFormulas(); const fallbackSheet = this.sheet(artifactId); - const addresses = elementIds.length + const requested = elementIds.length ? elementIds : this.observedCells().filter((cell) => cell.sheet === fallbackSheet.name).slice(0, 40).map((cell) => cell.address); - return addresses.map((elementId) => { - const target = this.target(elementId, fallbackSheet.name); + const targets: Array<{ sheet: ExcelJS.Worksheet; address: string; hint?: string }> = []; + const seen = new Set(); + for (let requestedIndex = 0; requestedIndex < requested.length; requestedIndex += 1) { + if (targets.length >= this.snapshotMaxCells) { + if (targets[0]) targets[0].hint = appendReadHint(targets[0].hint, "Additional requested ranges were omitted at the bounded read limit; request smaller subranges."); + break; + } + const elementId = requested[requestedIndex]; + const reference = parseReadReference(elementId); + if (!reference) throw new SpreadsheetBenchReadReferenceError(`Invalid SpreadsheetBench cell address or range: ${elementId}`); + let sheet: ExcelJS.Worksheet; + try { + sheet = this.sheet(reference.sheetName ?? fallbackSheet.name); + } catch (error) { + throw new SpreadsheetBenchReadReferenceError(error instanceof Error ? error.message : String(error)); + } + const remaining = this.snapshotMaxCells - targets.length; + const expansion = expandReadRange(reference.start, reference.end, remaining); + const firstTargetIndex = targets.length; + for (const address of expansion.addresses) { + const key = cellKey(sheet.name, address); + if (seen.has(key)) continue; + seen.add(key); + targets.push({ sheet, address }); + if (targets.length >= this.snapshotMaxCells) break; + } + if (expansion.truncated && targets[firstTargetIndex]) { + targets[firstTargetIndex].hint = `Requested ${sheet.name}!${reference.start}:${reference.end} contains ${expansion.totalCells} cells; read_range returned the first ${expansion.addresses.length} in row-major order. Request smaller subranges for the remainder.`; + } + if (targets.length >= this.snapshotMaxCells) { + if (requestedIndex < requested.length - 1 && targets[0]) { + targets[0].hint = appendReadHint(targets[0].hint, "Additional requested ranges were omitted at the bounded read limit; request smaller subranges."); + } + break; + } + } + return targets.map((target) => { const cell = target.sheet.getCell(target.address); return { id: target.address, @@ -428,6 +467,7 @@ class SpreadsheetBenchWorkbookRoomTools implements RoomTools { locked: this.isLocked(target.sheet.name, target.address) ? { by: "spreadsheetbench-nodeagent", reason: "managed workbook write" } : null, + ...(target.hint ? { hint: target.hint } : {}), }; }); } @@ -670,7 +710,7 @@ class SpreadsheetBenchWorkbookRoomTools implements RoomTools { const qualified = elementId.match(/^(?:'([^']+)'|([^!]+))!\s*(\$?[A-Z]{1,3}\$?[1-9][0-9]*)$/i); const sheet = qualified ? this.sheet((qualified[1] ?? qualified[2]).trim()) : this.sheet(fallbackSheet); const address = normalizeAddress(qualified ? qualified[3] : elementId); - if (!A1_RE.test(address)) throw new Error(`Invalid SpreadsheetBench cell address: ${elementId}`); + if (!parseAddress(address)) throw new Error(`Invalid SpreadsheetBench cell address: ${elementId}`); return { sheet, address }; } @@ -1056,7 +1096,31 @@ class BridgeWorkbookWorkflowController { return result; }); } - if (tool.name === "read_range" || tool.name === "search_sheet_context") { + if (tool.name === "read_range") { + const adapted = this.withExecution(tool, async (args, rt) => { + const record = asRecord(args) ?? {}; + const artifactId = await bridgeResolvedArtifactId(rt, record.artifactId, this.activeArtifactId); + try { + return await tool.execute({ ...record, artifactId }, rt); + } catch (error) { + if (!(error instanceof SpreadsheetBenchReadReferenceError)) throw error; + return { + ok: false, + error: "invalid_read_reference", + detail: error.message, + recovery: { + action: "retry_tool_call", + instruction: "Use a visible worksheet and Excel A1 cells within A1:XFD1048576; split large reads into smaller ranges.", + }, + }; + } + }); + return { + ...adapted, + description: `${adapted.description} In this workbook bridge, elementIds also accepts bounded A1 ranges such as B2:F20 and quoted sheet-qualified ranges such as 'Financial Overview'!B2:F20.`, + }; + } + if (tool.name === "search_sheet_context") { return this.withExecution(tool, async (args, rt) => { const record = asRecord(args) ?? {}; const artifactId = await bridgeResolvedArtifactId(rt, record.artifactId, this.activeArtifactId); @@ -1849,10 +1913,57 @@ function applyRoomValue(cell: ExcelJS.Cell, value: unknown): void { if (typeof record?.numFmt === "string") cell.numFmt = record.numFmt; } +function parseReadReference(value: string): { + sheetName?: string; + start: string; + end: string; +} | undefined { + const cleaned = value.trim().replace(/[,;]+$/, "").trim(); + const match = cleaned.match(READ_REFERENCE_RE); + if (!match) return undefined; + const quotedSheet = match[1]?.replace(/''/g, "'"); + const unquotedSheet = match[2]?.trim(); + const start = normalizeAddress(match[3]); + const end = normalizeAddress(match[4] ?? match[3]); + if (!parseAddress(start) || !parseAddress(end)) return undefined; + return { + ...(quotedSheet || unquotedSheet ? { sheetName: quotedSheet ?? unquotedSheet } : {}), + start, + end, + }; +} + +function expandReadRange(startText: string, endText: string, limit: number): { + addresses: string[]; + totalCells: number; + truncated: boolean; +} { + const start = parseAddress(startText); + const end = parseAddress(endText); + if (!start || !end) return { addresses: [], totalCells: 0, truncated: false }; + const minRow = Math.min(start.row, end.row); + const maxRow = Math.max(start.row, end.row); + const minCol = Math.min(start.col, end.col); + const maxCol = Math.max(start.col, end.col); + const totalCells = (maxRow - minRow + 1) * (maxCol - minCol + 1); + const boundedLimit = Math.max(0, Math.trunc(limit)); + const addresses: string[] = []; + for (let row = minRow; row <= maxRow && addresses.length < boundedLimit; row += 1) { + for (let col = minCol; col <= maxCol && addresses.length < boundedLimit; col += 1) { + addresses.push(addressFromPosition(row, col)); + } + } + return { addresses, totalCells, truncated: addresses.length < totalCells }; +} + +function appendReadHint(existing: string | undefined, addition: string): string { + return existing ? `${existing} ${addition}` : addition; +} + function expandRange(startText: string, endText: string, limit: number): string[] { const start = parseAddress(startText); const end = parseAddress(endText); - if (!start || !end) return A1_RE.test(startText) ? [normalizeAddress(startText)] : []; + if (!start || !end) return []; const minRow = Math.min(start.row, end.row); const maxRow = Math.max(start.row, end.row); const minCol = Math.min(start.col, end.col); @@ -1875,10 +1986,10 @@ function expandRange(startText: string, endText: string, limit: number): string[ function parseAddress(value: string): { row: number; col: number } | undefined { const match = normalizeAddress(value).match(/^([A-Z]{1,3})([1-9][0-9]*)$/); if (!match) return undefined; - return { - row: Number(match[2]), - col: match[1].split("").reduce((sum, char) => sum * 26 + char.charCodeAt(0) - 64, 0), - }; + const row = Number(match[2]); + const col = match[1].split("").reduce((sum, char) => sum * 26 + char.charCodeAt(0) - 64, 0); + if (row > EXCEL_MAX_ROW || col > EXCEL_MAX_COLUMN) return undefined; + return { row, col }; } function addressFromPosition(row: number, col: number): string { diff --git a/src/eval/spreadsheetBenchReportRepair.ts b/src/eval/spreadsheetBenchReportRepair.ts new file mode 100644 index 00000000..bad22a2e --- /dev/null +++ b/src/eval/spreadsheetBenchReportRepair.ts @@ -0,0 +1,253 @@ +import { resolve } from "node:path"; +import type { + SpreadsheetBenchRunnerCaseRun, + SpreadsheetBenchRunnerReport, + SpreadsheetBenchRunnerTaskResult, + SpreadsheetBenchSidecarFileEvidence, +} from "./spreadsheetBenchRunner"; + +export type SpreadsheetBenchRepairMergedReport = SpreadsheetBenchRunnerReport & { + composed: true; + repairMerge: { + schema: 1; + baseReportSha256: string; + repairReportSha256: string; + replacementTaskIds: string[]; + }; +}; + +export function assertSpreadsheetBenchMergeOutputPaths(args: { + reportOutput: string; + receiptOutput: string; + protectedPaths: string[]; +}): void { + const reportOutput = normalizedAbsolutePath(args.reportOutput); + const receiptOutput = normalizedAbsolutePath(args.receiptOutput); + if (reportOutput === receiptOutput) throw new Error("merge report and receipt outputs must be distinct files"); + const protectedPaths = new Set(args.protectedPaths.map(normalizedAbsolutePath)); + if (protectedPaths.has(reportOutput)) throw new Error("merge report output aliases an input or verified artifact"); + if (protectedPaths.has(receiptOutput)) throw new Error("merge receipt output aliases an input or verified artifact"); +} + +export function mergeSpreadsheetBenchRepairReport(args: { + base: SpreadsheetBenchRunnerReport; + repair: SpreadsheetBenchRunnerReport; + replacementTaskIds: string[]; + generatedAt: string; + outputRoot: string; + baseReportSha256: string; + repairReportSha256: string; +}): SpreadsheetBenchRepairMergedReport { + assertCompatibleReports(args.base, args.repair); + const replacementTaskIds = uniqueTaskIds(args.replacementTaskIds, "replacement task IDs"); + const replacementSet = new Set(replacementTaskIds); + const baseResults = uniqueByTaskId(args.base.results, "base results"); + const repairResults = uniqueByTaskId(args.repair.results, "repair results"); + const baseCaseRuns = uniqueByTaskId(args.base.caseRuns, "base case runs"); + const repairCaseRuns = uniqueByTaskId(args.repair.caseRuns, "repair case runs"); + + if (args.base.results.length !== args.base.taskCount || args.base.caseRuns.length !== args.base.taskCount) { + throw new Error("base report must contain exactly one result and one case run per task"); + } + if (args.repair.results.length !== replacementTaskIds.length || args.repair.caseRuns.length !== replacementTaskIds.length) { + throw new Error("repair report must contain exactly one result and one case run per replacement task"); + } + if (repairResults.size !== replacementSet.size || [...repairResults.keys()].some((taskId) => !replacementSet.has(taskId))) { + throw new Error("repair report task IDs must exactly match the replacement task IDs"); + } + + for (const taskId of replacementTaskIds) { + if (!baseResults.has(taskId) || !baseCaseRuns.has(taskId)) { + throw new Error(`replacement task is absent from the base report: ${taskId}`); + } + const result = repairResults.get(taskId); + const caseRun = repairCaseRuns.get(taskId); + if (!result || !caseRun) throw new Error(`repair result or case run is missing: ${taskId}`); + assertAuthenticRepairResult(result); + } + + const results = args.base.results.map((result) => repairResults.get(result.taskId) ?? result); + const caseRuns = args.base.caseRuns.map((caseRun) => repairCaseRuns.get(caseRun.taskId) ?? caseRun); + const passCount = results.filter((result) => result.score?.pass).length; + const casePassCount = caseRuns.filter((caseRun) => caseRun.pass).length; + const usage = aggregateUsage(results); + const { chunked: _chunked, chunkSize: _chunkSize, chunks: _chunks, ...base } = args.base as SpreadsheetBenchRunnerReport & { + chunked?: boolean; + chunkSize?: number; + chunks?: unknown[]; + }; + + return { + ...base, + generatedAt: args.generatedAt, + outputRoot: args.outputRoot, + composed: true, + taskCount: results.length, + passCount, + averageOverall: results.length + ? Number((results.reduce((sum, result) => sum + (result.score?.scores.overall ?? 0), 0) / results.length).toFixed(6)) + : 0, + caseCount: caseRuns.length, + caseRunCount: caseRuns.length, + casePassCount, + casePassRate: caseRuns.length ? Number((casePassCount / caseRuns.length).toFixed(6)) : 0, + attemptCount: results.length, + passRate: results.length ? Number((passCount / results.length).toFixed(6)) : 0, + retryStats: aggregateRetryStats(caseRuns), + stats: aggregateStats(results), + harness: { + toolPolicy: "agent_dir_only_until_candidate", + evaluatorAccess: "after_candidate_emit_only", + ...(args.base.harness.modelContextPolicy + ? { + modelContextPolicy: { + ...args.base.harness.modelContextPolicy, + selectedTaskCount: results.length, + }, + } + : {}), + budget: { + modelCalls: usage.calls, + inputTokens: usage.inputTokens, + outputTokens: usage.outputTokens, + providerCostUsd: usage.costUsd, + }, + }, + warnings: [...new Set([...args.base.warnings, ...args.repair.warnings])], + caseRuns, + results, + repairMerge: { + schema: 1, + baseReportSha256: args.baseReportSha256, + repairReportSha256: args.repairReportSha256, + replacementTaskIds, + }, + }; +} + +function assertCompatibleReports(base: SpreadsheetBenchRunnerReport, repair: SpreadsheetBenchRunnerReport): void { + if (base.schema !== 1 || repair.schema !== 1) throw new Error("SpreadsheetBench reports must use schema 1"); + if (base.mode !== "nodeagent-workbook" || repair.mode !== "nodeagent-workbook") { + throw new Error("repair merging requires nodeagent-workbook reports"); + } + if (base.stageRoot !== repair.stageRoot) throw new Error("base and repair stage roots differ"); + if (base.repeatCount !== 1 || repair.repeatCount !== 1) throw new Error("repair merging requires single-repeat reports"); + if (base.harness.toolPolicy !== "agent_dir_only_until_candidate" || repair.harness.toolPolicy !== base.harness.toolPolicy) { + throw new Error("base and repair tool policies differ"); + } + if (base.harness.evaluatorAccess !== "after_candidate_emit_only" || repair.harness.evaluatorAccess !== base.harness.evaluatorAccess) { + throw new Error("base and repair evaluator-access policies differ"); + } + if (canonicalJson(base.retryPolicy) !== canonicalJson(repair.retryPolicy)) { + throw new Error("base and repair retry policies differ"); + } + if (canonicalJson(contextPolicyWithoutSelection(base.harness.modelContextPolicy)) + !== canonicalJson(contextPolicyWithoutSelection(repair.harness.modelContextPolicy))) { + throw new Error("base and repair model-context policies differ"); + } +} + +function assertAuthenticRepairResult(result: SpreadsheetBenchRunnerTaskResult): void { + if (result.error) throw new Error(`repair result still has an execution error: ${result.taskId}`); + if (!result.candidateWorkbook || !isFileEvidence(result.scorerReceipt)) { + throw new Error(`repair result is missing a candidate or scorer receipt: ${result.taskId}`); + } + if (!isFileEvidence(result.sidecarEvidence?.candidateManifest) + || !isFileEvidence(result.sidecarEvidence?.nodeAgentReceipt) + || !isFileEvidence(result.sidecarEvidence?.nodeAgentTrace)) { + throw new Error(`repair result is missing NodeAgent receipt evidence: ${result.taskId}`); + } + if (!result.model?.name.trim() || result.model.calls < 1) { + throw new Error(`repair result has no authentic model call: ${result.taskId}`); + } +} + +function isFileEvidence(value: unknown): value is SpreadsheetBenchSidecarFileEvidence { + if (!value || typeof value !== "object" || Array.isArray(value)) return false; + const evidence = value as Partial; + return typeof evidence.path === "string" + && evidence.path.trim().length > 0 + && typeof evidence.sha256 === "string" + && /^[a-f0-9]{64}$/i.test(evidence.sha256) + && Number.isSafeInteger(evidence.bytes) + && (evidence.bytes ?? -1) >= 0; +} + +function contextPolicyWithoutSelection(policy: SpreadsheetBenchRunnerReport["harness"]["modelContextPolicy"]): unknown { + if (!policy) return null; + const { selectedTaskCount: _selectedTaskCount, ...contract } = policy; + return contract; +} + +function canonicalJson(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + if (value && typeof value === "object") { + return `{${Object.entries(value as Record) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, entry]) => `${JSON.stringify(key)}:${canonicalJson(entry)}`) + .join(",")}}`; + } + return JSON.stringify(value) ?? "undefined"; +} + +function normalizedAbsolutePath(path: string): string { + return resolve(path).replace(/\\/g, "/").toLowerCase(); +} + +function uniqueTaskIds(values: string[], label: string): string[] { + const normalized = values.map((value) => value.trim()); + if (normalized.some((value) => !value)) throw new Error(`${label} contain an empty task ID`); + if (new Set(normalized).size !== normalized.length) throw new Error(`${label} contain duplicates`); + return normalized; +} + +function uniqueByTaskId(values: T[], label: string): Map { + const result = new Map(); + for (const value of values) { + if (!value.taskId.trim()) throw new Error(`${label} contain an empty task ID`); + if (result.has(value.taskId)) throw new Error(`${label} contain duplicate task ID ${value.taskId}`); + result.set(value.taskId, value); + } + return result; +} + +function aggregateUsage(results: SpreadsheetBenchRunnerTaskResult[]) { + return { + calls: results.reduce((sum, result) => sum + (result.model?.calls ?? 0), 0), + inputTokens: results.reduce((sum, result) => sum + (result.model?.usage.inputTokens ?? 0), 0), + outputTokens: results.reduce((sum, result) => sum + (result.model?.usage.outputTokens ?? 0), 0), + costUsd: Number(results.reduce((sum, result) => sum + (result.model?.costUsd ?? 0), 0).toFixed(8)), + }; +} + +function aggregateRetryStats(caseRuns: SpreadsheetBenchRunnerCaseRun[]): SpreadsheetBenchRunnerReport["retryStats"] { + return { + retriedCaseRunCount: caseRuns.filter((run) => run.attempts.length > 1).length, + retryAttemptCount: caseRuns.reduce((sum, run) => sum + Math.max(0, run.attempts.length - 1), 0), + passedAfterRetryCount: caseRuns.filter((run) => run.pass && run.attempts.length > 1).length, + exhaustedCaseRunCount: caseRuns.filter((run) => run.stopReason === "retry_exhausted").length, + }; +} + +function aggregateStats(results: SpreadsheetBenchRunnerTaskResult[]): SpreadsheetBenchRunnerReport["stats"] { + const latencies = results.map((result) => result.timingsMs.total).sort((a, b) => a - b); + const failureCounts: Record = {}; + for (const result of results) { + if (!result.error) continue; + failureCounts[result.error.phase] = (failureCounts[result.error.phase] ?? 0) + 1; + } + return { + latencyMs: { + p50: percentile(latencies, 0.5), + p95: percentile(latencies, 0.95), + max: latencies.at(-1) ?? 0, + }, + failureCounts, + }; +} + +function percentile(values: number[], quantile: number): number { + if (values.length === 0) return 0; + const index = Math.min(values.length - 1, Math.ceil(values.length * quantile) - 1); + return values[index] ?? 0; +} diff --git a/src/nodeagent/models/qualityFailover.ts b/src/nodeagent/models/qualityFailover.ts index 25c85aa5..1da12f95 100644 --- a/src/nodeagent/models/qualityFailover.ts +++ b/src/nodeagent/models/qualityFailover.ts @@ -374,7 +374,7 @@ export function classifyQualityFailoverProviderError(error: unknown): ProviderFa if (/\b(?:401|unauthori[sz]ed|invalid api key|authentication required)\b/i.test(detail)) { return globalProviderFailure("auth", "provider_auth_required", detail); } - if (/\b(?:402|insufficient credits?|(?:daily|monthly|global)?\s*quota (?:exhausted|exceeded))\b/i.test(detail)) { + if (/\b(?:402|insufficient credits?|(?:daily|monthly|global)?\s*quota (?:exhausted|exceeded)|free-models-per-day(?:-high-balance)?)\b/i.test(detail)) { return globalProviderFailure("quota", "provider_quota_exhausted", detail); } if (/\b(?:403|forbidden|content policy|policy violation|provider_(?:egress|route)_blocked)\b/i.test(detail)) { diff --git a/src/nodeagent/traces/traceRedaction.ts b/src/nodeagent/traces/traceRedaction.ts index f73460a3..b04bda94 100644 --- a/src/nodeagent/traces/traceRedaction.ts +++ b/src/nodeagent/traces/traceRedaction.ts @@ -9,20 +9,38 @@ export function redactTraceText(value: string, replacement = "[redacted]"): stri } export function stableTraceJson(value: unknown): string { - const seen = new WeakSet(); - const normalize = (input: unknown): unknown => { + const ancestors = new WeakSet(); + const normalize = (input: unknown, key: string): unknown => { if (typeof input === "string") return redactTraceText(input); if (input === null || typeof input !== "object") return input; - if (seen.has(input)) return "[circular]"; - seen.add(input); - if (Array.isArray(input)) return input.map((item) => normalize(item)); - return Object.fromEntries( - Object.entries(input as Record) - .sort(([a], [b]) => a.localeCompare(b)) - .map(([key, entry]) => [key, normalize(entry)]), - ); + + if (ancestors.has(input)) return "[circular]"; + const toJSON = (input as { toJSON?: unknown }).toJSON; + if (typeof toJSON === "function") { + ancestors.add(input); + try { + const serialized = toJSON.call(input, key); + if (serialized !== input) return normalize(serialized, key); + } catch { + // Keep trace hashing total for objects whose custom serializer fails. + } finally { + ancestors.delete(input); + } + } + + ancestors.add(input); + try { + if (Array.isArray(input)) return input.map((item, index) => normalize(item, String(index))); + return Object.fromEntries( + Object.entries(input as Record) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([entryKey, entry]) => [entryKey, normalize(entry, entryKey)]), + ); + } finally { + ancestors.delete(input); + } }; - return JSON.stringify(normalize(value)); + return JSON.stringify(normalize(value, "")); } export function stableTraceHash(value: unknown): string { diff --git a/tests/modelAdapterProviderNeutralRoute.test.ts b/tests/modelAdapterProviderNeutralRoute.test.ts index 13556d13..72abacd6 100644 --- a/tests/modelAdapterProviderNeutralRoute.test.ts +++ b/tests/modelAdapterProviderNeutralRoute.test.ts @@ -128,7 +128,7 @@ describe("provider-neutral NodeAgent free-first routing", () => { it("falls back to direct Gemini only after provider-wide OpenRouter quota exhaustion", async () => { process.env.GOOGLE_GENERATIVE_AI_API_KEY = "test-google-key"; generateTextMock - .mockRejectedValueOnce(new Error("Provider request failed 429: free-models-per-day-high-balance rate limit exceeded")) + .mockRejectedValueOnce(new Error("AI_RetryError: Failed after 3 attempts. Last error: Rate limit exceeded: free-models-per-day-high-balance.")) .mockImplementationOnce(async (options: { model: { id: string } }) => successfulTurn(options.model.id)); const { route, step } = await runNeutralRoute(); @@ -146,7 +146,7 @@ describe("provider-neutral NodeAgent free-first routing", () => { primary: { route: "openrouter/free-auto", outcome: "provider_wide_exhausted", - reason: "provider_free_quota_exhausted", + reason: "provider_quota_exhausted", qualityFailover: { stopReason: "global_provider_failure", terminalFailure: { providerFailureScope: "global", providerFailureCategory: "quota" }, diff --git a/tests/nodeagentTraceSpine.test.ts b/tests/nodeagentTraceSpine.test.ts index ef553442..72ef3c49 100644 --- a/tests/nodeagentTraceSpine.test.ts +++ b/tests/nodeagentTraceSpine.test.ts @@ -85,6 +85,25 @@ describe("nodeagent trace spine", () => { expect(trace.trigger.prompt).not.toContain("banker@example.com"); expect(redactTraceText("send to banker@example.com")).toContain("[redacted]"); expect(stableTraceHash({ b: 2, a: 1 })).toBe(stableTraceHash({ a: 1, b: 2 })); + const datedResult = { cell: { value: new Date("2026-07-14T12:34:56.000Z") } }; + expect(stableTraceHash(datedResult)).toBe( + stableTraceHash(JSON.parse(JSON.stringify(datedResult))), + ); + const sharedCellStyle = { numFmt: "0.0x", font: { bold: true } }; + const sharedResult = { cells: [{ style: sharedCellStyle }, { style: sharedCellStyle }] }; + expect(stableTraceHash(sharedResult)).toBe( + stableTraceHash(JSON.parse(JSON.stringify(sharedResult))), + ); + const keySensitiveResult = { + result: { + toJSON(key: string) { + return { serializedFor: key }; + }, + }, + }; + expect(stableTraceHash(keySensitiveResult)).toBe( + stableTraceHash(JSON.parse(JSON.stringify(keySensitiveResult))), + ); expect(traceExcellenceLevel(trace)).toBe(3); expect(summarizeTrace(trace)).toContain("L3 evidence links"); expect(trace.final.status).toBe("completed"); diff --git a/tests/qualityFailover.test.ts b/tests/qualityFailover.test.ts index 29e27e1b..30544fd0 100644 --- a/tests/qualityFailover.test.ts +++ b/tests/qualityFailover.test.ts @@ -174,6 +174,12 @@ describe("quality-aware bounded failover", () => { reason: "provider_free_quota_exhausted", category: "quota", }, + { + label: "wrapped quota", + error: new Error("AI_RetryError: Failed after 3 attempts. Last error: Rate limit exceeded: free-models-per-day-high-balance."), + reason: "provider_quota_exhausted", + category: "quota", + }, { label: "policy", error: new Error("provider_route_blocked:provider_not_allowed"), diff --git a/tests/spreadsheetBenchNodeAgentBridge.test.ts b/tests/spreadsheetBenchNodeAgentBridge.test.ts index 253333dc..7d84d4b6 100644 --- a/tests/spreadsheetBenchNodeAgentBridge.test.ts +++ b/tests/spreadsheetBenchNodeAgentBridge.test.ts @@ -395,6 +395,59 @@ describe("SpreadsheetBench canonical NodeAgent bridge", () => { expect(emitted.getWorksheet("Cover")?.getCell("A1").value).toBe("Cover sheet"); }); + it("expands bounded quoted sheet ranges without aborting the NodeAgent frame", async () => { + const root = tempRoot(); + const task = await stagedMultiSheetTask(root); + const candidate = join(root, "output", "range-read.xlsx"); + + const receipt = await runSpreadsheetBenchNodeAgentBridge({ + agentManifestPath: task.agentManifest, + candidateWorkbookPath: candidate, + model: boundedRangeReadModel(), + traceId: "trace_sbench_bounded_range_read", + maxSteps: 8, + snapshotMaxCells: 20, + now: () => 3_250, + }); + + expect(receipt.frame.runtimeError).toBeUndefined(); + expect(receipt.outcome).toMatchObject({ status: "completed", changedCellCount: 1, finalVerificationStatus: "passed" }); + expect(receipt.frame.agentResult.trace.find((event) => event.tool === "read_range")?.result).toEqual([ + expect.objectContaining({ id: "B1" }), + expect.objectContaining({ id: "C1" }), + expect.objectContaining({ id: "D1" }), + expect.objectContaining({ id: "A1" }), + ]); + }); + + it("returns recoverable feedback for invalid Excel bounds and discloses truncated range reads", async () => { + const root = tempRoot(); + const task = await stagedMultiSheetTask(root); + const receipt = await runSpreadsheetBenchNodeAgentBridge({ + agentManifestPath: task.agentManifest, + candidateWorkbookPath: join(root, "output", "range-recovery.xlsx"), + model: recoveringBoundedRangeModel(), + traceId: "trace_sbench_range_recovery", + maxSteps: 8, + snapshotMaxCells: 3, + now: () => 3_500, + }); + + const reads = receipt.frame.agentResult.trace.filter((event) => event.tool === "read_range"); + expect(receipt.frame.runtimeError).toBeUndefined(); + expect(reads[0]?.result).toMatchObject({ + ok: false, + error: "invalid_read_reference", + recovery: { action: "retry_tool_call" }, + }); + expect(reads[1]?.result).toEqual([ + expect.objectContaining({ id: "A1", hint: expect.stringContaining("contains 4 cells") }), + expect.objectContaining({ id: "B1" }), + expect.objectContaining({ id: "C1" }), + ]); + expect(receipt.outcome).toMatchObject({ status: "completed", changedCellCount: 1, finalVerificationStatus: "passed" }); + }); + it("keeps workbook-wide average repairs open until every worksheet passes post-write verification", async () => { const root = tempRoot(); const task = await stagedWorkbookWideAverageTask(root); @@ -781,6 +834,46 @@ function preferredSheetModel(): AgentModel { }; } +function boundedRangeReadModel(): AgentModel { + let callIndex = 0; + const operation = { elementId: "B1", formula: "AVERAGE(C1:D1)", result: 15 }; + return { + name: "scripted-bounded-range-read", + async next(): Promise { + const id = `bounded-range-${++callIndex}`; + if (callIndex === 1) return step(id, "inspect_workbook", { instruction: "Audit the workbook.", maxCells: 40 }); + if (callIndex === 2) { + return step(id, "read_range", { + artifactId: "Metrics", + elementIds: ["'Metrics'!$B$1:$D$1", "Metrics!A1,"], + }); + } + if (callIndex === 3) return step(id, "verify_workbook", { instruction: "Audit the workbook.", operations: [operation], afterWrite: false }); + if (callIndex === 4) return step(id, "write_locked_cells", { reason: "repair incorrect average", ops: [operation] }); + if (callIndex === 5) return step(id, "verify_workbook", { instruction: "Audit the workbook.", operations: [operation], afterWrite: true }); + return { text: "Average formula repaired and verified after a bounded range read.", toolCalls: [], done: true }; + }, + }; +} + +function recoveringBoundedRangeModel(): AgentModel { + let callIndex = 0; + const operation = { elementId: "B1", formula: "AVERAGE(C1:D1)", result: 15 }; + return { + name: "scripted-recovering-bounded-range", + async next(): Promise { + const id = `recovering-range-${++callIndex}`; + if (callIndex === 1) return step(id, "inspect_workbook", { instruction: "Audit the workbook.", maxCells: 40 }); + if (callIndex === 2) return step(id, "read_range", { artifactId: "Metrics", elementIds: ["XFE1"] }); + if (callIndex === 3) return step(id, "read_range", { artifactId: "Metrics", elementIds: ["'Metrics'!A1:D1"] }); + if (callIndex === 4) return step(id, "verify_workbook", { instruction: "Audit the workbook.", operations: [operation], afterWrite: false }); + if (callIndex === 5) return step(id, "write_locked_cells", { reason: "repair incorrect average", ops: [operation] }); + if (callIndex === 6) return step(id, "verify_workbook", { instruction: "Audit the workbook.", operations: [operation], afterWrite: true }); + return { text: "Recovered from an invalid range and completed the verified repair.", toolCalls: [], done: true }; + }, + }; +} + function workbookWideAverageModel(failAfterProof = false): AgentModel { let callIndex = 0; const wacc = [ diff --git a/tests/spreadsheetBenchReportRepair.test.ts b/tests/spreadsheetBenchReportRepair.test.ts new file mode 100644 index 00000000..cbaa216a --- /dev/null +++ b/tests/spreadsheetBenchReportRepair.test.ts @@ -0,0 +1,213 @@ +import { describe, expect, it } from "vitest"; +import { + assertSpreadsheetBenchMergeOutputPaths, + mergeSpreadsheetBenchRepairReport, +} from "../src/eval/spreadsheetBenchReportRepair"; +import type { + SpreadsheetBenchRunnerCaseRun, + SpreadsheetBenchRunnerReport, + SpreadsheetBenchRunnerTaskResult, +} from "../src/eval/spreadsheetBenchRunner"; + +describe("SpreadsheetBench NodeAgent repair report merge", () => { + it("replaces only allowlisted tasks and recomputes aggregate proof fields", () => { + const baseOne = taskResult("Debugging/01_01", "openrouter/free-auto", 0, 0.1); + const baseTwo = taskResult("Debugging/01_02", "cohere/north-mini-code:free", 2, 0.2); + const repaired = taskResult("Debugging/01_01", "gemini-3-flash-preview", 3, 0.8); + const merged = mergeSpreadsheetBenchRepairReport({ + base: report([baseOne, baseTwo], "base-run"), + repair: report([repaired], "repair-run"), + replacementTaskIds: ["Debugging/01_01"], + generatedAt: "2026-07-14T00:00:00.000Z", + outputRoot: "merged-run", + baseReportSha256: "a".repeat(64), + repairReportSha256: "b".repeat(64), + }); + + expect(merged.composed).toBe(true); + expect(merged.results.map((result) => result.model?.name)).toEqual([ + "gemini-3-flash-preview", + "cohere/north-mini-code:free", + ]); + expect(merged.results[1]).toBe(baseTwo); + expect(merged.averageOverall).toBe(0.5); + expect(merged.harness.budget).toEqual({ + modelCalls: 5, + inputTokens: 50, + outputTokens: 10, + providerCostUsd: 0.05, + }); + expect(merged.repairMerge.replacementTaskIds).toEqual(["Debugging/01_01"]); + }); + + it("rejects a repair report that does not exactly match the replacement allowlist", () => { + expect(() => mergeSpreadsheetBenchRepairReport({ + base: report([taskResult("Debugging/01_01", "openrouter/free-auto", 0, 0.1)], "base-run"), + repair: report([taskResult("Debugging/01_02", "gemini-3-flash-preview", 1, 0.8)], "repair-run"), + replacementTaskIds: ["Debugging/01_01"], + generatedAt: "2026-07-14T00:00:00.000Z", + outputRoot: "merged-run", + baseReportSha256: "a".repeat(64), + repairReportSha256: "b".repeat(64), + })).toThrow(/exactly match/); + }); + + it("rejects replacement results with no authentic model call", () => { + const taskId = "Debugging/01_01"; + expect(() => mergeSpreadsheetBenchRepairReport({ + base: report([taskResult(taskId, "openrouter/free-auto", 0, 0.1)], "base-run"), + repair: report([taskResult(taskId, "openrouter/free-auto", 0, 0.8)], "repair-run"), + replacementTaskIds: [taskId], + generatedAt: "2026-07-14T00:00:00.000Z", + outputRoot: "merged-run", + baseReportSha256: "a".repeat(64), + repairReportSha256: "b".repeat(64), + })).toThrow(/no authentic model call/); + }); + + it("rejects malformed NodeAgent sidecar evidence before claiming authentic calls", () => { + const taskId = "Debugging/01_01"; + const repaired = taskResult(taskId, "gemini-3-flash-preview", 1, 0.8); + repaired.sidecarEvidence!.nodeAgentReceipt = {} as never; + expect(() => mergeSpreadsheetBenchRepairReport({ + base: report([taskResult(taskId, "openrouter/free-auto", 0, 0.1)], "base-run"), + repair: report([repaired], "repair-run"), + replacementTaskIds: [taskId], + generatedAt: "2026-07-14T00:00:00.000Z", + outputRoot: "merged-run", + baseReportSha256: "a".repeat(64), + repairReportSha256: "b".repeat(64), + })).toThrow(/missing NodeAgent receipt evidence/); + }); + + it("rejects repair execution-policy drift that a merged base policy would conceal", () => { + const taskId = "Debugging/01_01"; + const base = report([taskResult(taskId, "openrouter/free-auto", 0, 0.1)], "base-run"); + const repair = report([taskResult(taskId, "gemini-3-flash-preview", 1, 0.8)], "repair-run"); + repair.harness.modelContextPolicy!.snapshotMaxCells = 1_200; + expect(() => mergeSpreadsheetBenchRepairReport({ + base, + repair, + replacementTaskIds: [taskId], + generatedAt: "2026-07-14T00:00:00.000Z", + outputRoot: "merged-run", + baseReportSha256: "a".repeat(64), + repairReportSha256: "b".repeat(64), + })).toThrow(/model-context policies differ/); + + const retryDrift = report([taskResult(taskId, "gemini-3-flash-preview", 1, 0.8)], "repair-run"); + retryDrift.retryPolicy.maxRetries = 1; + expect(() => mergeSpreadsheetBenchRepairReport({ + base, + repair: retryDrift, + replacementTaskIds: [taskId], + generatedAt: "2026-07-14T00:00:00.000Z", + outputRoot: "merged-run", + baseReportSha256: "a".repeat(64), + repairReportSha256: "b".repeat(64), + })).toThrow(/retry policies differ/); + }); + + it("rejects output aliases against receipts, reports, and verified artifacts", () => { + expect(() => assertSpreadsheetBenchMergeOutputPaths({ + reportOutput: "proof/merged.json", + receiptOutput: "proof/merged.json", + protectedPaths: [], + })).toThrow(/must be distinct/); + expect(() => assertSpreadsheetBenchMergeOutputPaths({ + reportOutput: "proof/merged.json", + receiptOutput: "proof/receipt.json", + protectedPaths: ["proof/merged.json"], + })).toThrow(/aliases an input or verified artifact/); + }); +}); + +function taskResult(taskId: string, modelName: string, calls: number, overall: number): SpreadsheetBenchRunnerTaskResult { + const artifact = taskId.replaceAll("/", "_"); + return { + taskId, + track: "spreadsheetbench-v2", + category: "Debugging", + mode: "nodeagent-workbook", + attemptIndex: 1, + repeatIndex: 1, + tryIndex: 1, + taskDir: `tasks/${artifact}`, + agentManifest: `tasks/${artifact}/agent/task.json`, + evaluatorManifest: `tasks/${artifact}/evaluator/task.json`, + candidateWorkbook: `${artifact}/candidate.xlsx`, + sidecarEvidence: { + candidateManifest: evidence(`${artifact}/candidate-manifest.json`), + nodeAgentReceipt: evidence(`${artifact}/nodeagent-workbook-receipt.json`), + nodeAgentTrace: evidence(`${artifact}/nodeagent-workbook-trace.json`), + }, + scorerReceipt: evidence(`${artifact}/scorer-receipt.json`), + score: { pass: overall === 1, scores: { overall } } as SpreadsheetBenchRunnerTaskResult["score"], + model: { + name: modelName, + calls, + usage: { inputTokens: calls * 10, outputTokens: calls * 2, cachedInputTokens: 0 }, + costUsd: calls * 0.01, + }, + timingsMs: { candidateGeneration: 10, scoring: 5, total: 15 }, + trajectory: [], + }; +} + +function report(results: SpreadsheetBenchRunnerTaskResult[], outputRoot: string): SpreadsheetBenchRunnerReport { + const caseRuns: SpreadsheetBenchRunnerCaseRun[] = results.map((result) => ({ + taskId: result.taskId, + taskDir: result.taskDir, + repeatIndex: 1, + attempts: [1], + finalAttemptIndex: 1, + pass: result.score?.pass ?? false, + stopReason: result.score?.pass ? "passed" : "failed_score", + bestOverall: result.score?.scores.overall ?? 0, + })); + return { + schema: 1, + generatedAt: "2026-07-14T00:00:00.000Z", + stageRoot: "staged-v2-full", + outputRoot, + mode: "nodeagent-workbook", + taskCount: results.length, + passCount: results.filter((result) => result.score?.pass).length, + averageOverall: 0, + caseCount: caseRuns.length, + caseRunCount: caseRuns.length, + casePassCount: 0, + casePassRate: 0, + repeatCount: 1, + attemptCount: results.length, + passRate: 0, + retryPolicy: { maxRetries: 0, retryOn: ["candidate_generation", "scoring"], stopOnPass: true }, + retryStats: { + retriedCaseRunCount: 0, + retryAttemptCount: 0, + passedAfterRetryCount: 0, + exhaustedCaseRunCount: 0, + }, + stats: { latencyMs: { p50: 0, p95: 0, max: 0 }, failureCounts: {} }, + harness: { + toolPolicy: "agent_dir_only_until_candidate", + evaluatorAccess: "after_candidate_emit_only", + modelContextPolicy: { + batchSize: 1, + snapshotMaxCells: 6000, + snapshotMaxCellChars: null, + instructionMaxChars: null, + repairAttempts: 2, + selectedTaskCount: results.length, + }, + budget: { modelCalls: 0, inputTokens: 0, outputTokens: 0, providerCostUsd: 0 }, + }, + warnings: [], + caseRuns, + results, + }; +} + +function evidence(path: string) { + return { path, sha256: "c".repeat(64), bytes: 1 }; +} From 3c40791df79b11e1f034af5e428e409198cdcc44 Mon Sep 17 00:00:00 2001 From: homen Date: Wed, 15 Jul 2026 03:14:41 -0700 Subject: [PATCH 07/12] fix(nodeagent): recover from cooling free routes --- src/nodeagent/models/adapter.ts | 36 ++++++++++------- .../modelAdapterProviderNeutralRoute.test.ts | 39 ++++++++++++++++++- 2 files changed, 59 insertions(+), 16 deletions(-) diff --git a/src/nodeagent/models/adapter.ts b/src/nodeagent/models/adapter.ts index 57dfac6d..ea43a902 100644 --- a/src/nodeagent/models/adapter.ts +++ b/src/nodeagent/models/adapter.ts @@ -166,7 +166,7 @@ type ProviderNeutralRouteReceipt = ProviderRouteReceipt & { requestedModel: typeof NODEAGENT_FREE_AUTO_MODEL; primary: { route: typeof OPENROUTER_FREE_AUTO_MODEL; - outcome: "accepted" | "provider_wide_exhausted"; + outcome: "accepted" | "provider_wide_exhausted" | "free_routes_cooling"; reason?: string; qualityFailover?: QualityFailoverReceipt; }; @@ -378,14 +378,14 @@ async function generateProviderNeutralAgentText(args: { }), }; } catch (primaryError) { - const exhaustion = openRouterProviderWideExhaustion(primaryError); - if (!exhaustion || args.signal?.aborted) throw primaryError; + const unavailable = openRouterPrimaryUnavailable(primaryError); + if (!unavailable || args.signal?.aborted) throw primaryError; const configuredKey = envValue("GOOGLE_GENERATIVE_AI_API_KEY"); if (!configuredKey) { throw providerNeutralRecoveryError( primaryError, - `${exhaustion.reason}; direct Google fallback is unavailable because GOOGLE_GENERATIVE_AI_API_KEY is not configured`, + `${unavailable.reason}; direct Google fallback is unavailable because GOOGLE_GENERATIVE_AI_API_KEY is not configured`, ); } @@ -395,7 +395,7 @@ async function generateProviderNeutralAgentText(args: { if (getProviderForModel(fallbackModel) !== "gemini") { throw providerNeutralRecoveryError( primaryError, - `${exhaustion.reason}; NODEAGENT_FREE_AUTO_GOOGLE_MODEL must resolve to a direct Gemini model`, + `${unavailable.reason}; NODEAGENT_FREE_AUTO_GOOGLE_MODEL must resolve to a direct Gemini model`, ); } @@ -406,7 +406,7 @@ async function generateProviderNeutralAgentText(args: { } catch (policyError) { throw providerNeutralRecoveryError( primaryError, - `${exhaustion.reason}; direct Google fallback blocked before provider call: ${shortProviderError(policyError)}`, + `${unavailable.reason}; direct Google fallback blocked before provider call: ${shortProviderError(policyError)}`, policyError, ); } @@ -426,22 +426,23 @@ async function generateProviderNeutralAgentText(args: { providerRoute: providerNeutralRouteReceipt({ selectedReceipt: providerRoute, resolvedModel: fallbackModel, - primaryOutcome: "provider_wide_exhausted", - primaryReason: exhaustion.reason, - primaryQualityFailover: exhaustion.receipt, + primaryOutcome: unavailable.outcome, + primaryReason: unavailable.reason, + primaryQualityFailover: unavailable.receipt, }), }; } catch (fallbackError) { throw providerNeutralRecoveryError( primaryError, - `${exhaustion.reason}; direct Google fallback failed: ${shortProviderError(fallbackError)}`, + `${unavailable.reason}; direct Google fallback failed: ${shortProviderError(fallbackError)}`, fallbackError, ); } } } -function openRouterProviderWideExhaustion(error: unknown): { +function openRouterPrimaryUnavailable(error: unknown): { + outcome: "provider_wide_exhausted" | "free_routes_cooling"; reason: string; receipt?: QualityFailoverReceipt; } | undefined { @@ -450,12 +451,17 @@ function openRouterProviderWideExhaustion(error: unknown): { if (terminal?.failureClass === "provider" && terminal.providerFailureScope === "global" && terminal.providerFailureCategory === "quota") { - return { reason: terminal.reason, receipt: error.receipt }; + return { outcome: "provider_wide_exhausted", reason: terminal.reason, receipt: error.receipt }; } return undefined; } + if (/openrouter\/free-auto candidates cooling down/i.test(shortProviderError(error))) { + return { outcome: "free_routes_cooling", reason: "provider_free_routes_cooling" }; + } const reason = providerNonRetryableReason(error); - return reason && /quota|credit/i.test(reason) ? { reason } : undefined; + return reason && /quota|credit/i.test(reason) + ? { outcome: "provider_wide_exhausted", reason } + : undefined; } function providerNeutralRecoveryError( @@ -463,7 +469,7 @@ function providerNeutralRecoveryError( detail: string, cause: unknown = primaryError, ): Error { - const message = `${NODEAGENT_FREE_AUTO_MODEL} could not recover after OpenRouter provider-wide exhaustion: ${detail}`; + const message = `${NODEAGENT_FREE_AUTO_MODEL} could not recover after OpenRouter free-route unavailability: ${detail}`; if (primaryError instanceof QualityFailoverError) { return new QualityFailoverError(message, primaryError.receipt, cause, primaryError.usage); } @@ -473,7 +479,7 @@ function providerNeutralRecoveryError( function providerNeutralRouteReceipt(args: { selectedReceipt: ProviderRouteReceipt; resolvedModel: string; - primaryOutcome: "accepted" | "provider_wide_exhausted"; + primaryOutcome: "accepted" | "provider_wide_exhausted" | "free_routes_cooling"; primaryReason?: string; primaryQualityFailover?: QualityFailoverReceipt; }): ProviderNeutralRouteReceipt { diff --git a/tests/modelAdapterProviderNeutralRoute.test.ts b/tests/modelAdapterProviderNeutralRoute.test.ts index 72abacd6..f8b19d87 100644 --- a/tests/modelAdapterProviderNeutralRoute.test.ts +++ b/tests/modelAdapterProviderNeutralRoute.test.ts @@ -5,7 +5,10 @@ import { NODEAGENT_FREE_AUTO_MODEL, resolveModelAlias, } from "../src/nodeagent/models/modelCatalog"; -import { resetOpenRouterFreeRouteHealth } from "../src/nodeagent/models/openRouterFreeModels"; +import { + recordOpenRouterFreeRouteOutcome, + resetOpenRouterFreeRouteHealth, +} from "../src/nodeagent/models/openRouterFreeModels"; import { QualityFailoverError } from "../src/nodeagent/models/qualityFailover"; const generateTextMock = vi.hoisted(() => vi.fn()); @@ -168,6 +171,40 @@ describe("provider-neutral NodeAgent free-first routing", () => { expect(step.providerRoute?.basis).toContain("billing:account_free_tier:not_asserted"); }); + it("falls back to direct Gemini when every governed free route is cooling", async () => { + process.env.GOOGLE_GENERATIVE_AI_API_KEY = "test-google-key"; + recordOpenRouterFreeRouteOutcome({ + modelId: OPENROUTER_MODEL, + ok: false, + latencyMs: 1_000, + error: new Error("provider route unavailable"), + }); + + const { route, step } = await runNeutralRoute(); + + expect(generateTextMock.mock.calls.map((call) => call[0].model)).toEqual([ + { provider: "google", id: GOOGLE_MODEL }, + ]); + expect(route.name).toBe(GOOGLE_MODEL); + expect(step.providerRoute).toMatchObject({ + requestedModel: NODEAGENT_FREE_AUTO_MODEL, + resolvedModel: GOOGLE_MODEL, + provider: "gemini", + providerNeutral: { + primary: { + route: "openrouter/free-auto", + outcome: "free_routes_cooling", + reason: "provider_free_routes_cooling", + }, + selected: { + route: "google_direct", + model: GOOGLE_MODEL, + billing: { free: false, classification: "catalog_priced" }, + }, + }, + }); + }); + it("does not cross providers for an ordinary candidate-scoped OpenRouter failure", async () => { process.env.GOOGLE_GENERATIVE_AI_API_KEY = "test-google-key"; generateTextMock.mockRejectedValueOnce(new Error("Provider request failed 503: route unavailable")); From d39803a2c688f136f76d193bfd210eeeeffd8477 Mon Sep 17 00:00:00 2001 From: homen Date: Wed, 15 Jul 2026 03:56:26 -0700 Subject: [PATCH 08/12] fix(nodeagent): recover after free ladder exhaustion --- src/nodeagent/models/adapter.ts | 15 ++++++++-- .../modelAdapterProviderNeutralRoute.test.ts | 29 +++++++++++++++++-- 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/src/nodeagent/models/adapter.ts b/src/nodeagent/models/adapter.ts index ea43a902..13c9ce46 100644 --- a/src/nodeagent/models/adapter.ts +++ b/src/nodeagent/models/adapter.ts @@ -166,7 +166,7 @@ type ProviderNeutralRouteReceipt = ProviderRouteReceipt & { requestedModel: typeof NODEAGENT_FREE_AUTO_MODEL; primary: { route: typeof OPENROUTER_FREE_AUTO_MODEL; - outcome: "accepted" | "provider_wide_exhausted" | "free_routes_cooling"; + outcome: "accepted" | "provider_wide_exhausted" | "free_routes_cooling" | "free_routes_exhausted"; reason?: string; qualityFailover?: QualityFailoverReceipt; }; @@ -442,7 +442,7 @@ async function generateProviderNeutralAgentText(args: { } function openRouterPrimaryUnavailable(error: unknown): { - outcome: "provider_wide_exhausted" | "free_routes_cooling"; + outcome: "provider_wide_exhausted" | "free_routes_cooling" | "free_routes_exhausted"; reason: string; receipt?: QualityFailoverReceipt; } | undefined { @@ -453,6 +453,15 @@ function openRouterPrimaryUnavailable(error: unknown): { && terminal.providerFailureCategory === "quota") { return { outcome: "provider_wide_exhausted", reason: terminal.reason, receipt: error.receipt }; } + if (terminal?.providerFailureScope === "global") return undefined; + if (["no_candidates", "candidates_exhausted", "attempt_budget", "time_budget", "cooldown"] + .includes(error.receipt.stopReason)) { + return { + outcome: "free_routes_exhausted", + reason: `provider_free_routes_${error.receipt.stopReason}`, + receipt: error.receipt, + }; + } return undefined; } if (/openrouter\/free-auto candidates cooling down/i.test(shortProviderError(error))) { @@ -479,7 +488,7 @@ function providerNeutralRecoveryError( function providerNeutralRouteReceipt(args: { selectedReceipt: ProviderRouteReceipt; resolvedModel: string; - primaryOutcome: "accepted" | "provider_wide_exhausted" | "free_routes_cooling"; + primaryOutcome: "accepted" | "provider_wide_exhausted" | "free_routes_cooling" | "free_routes_exhausted"; primaryReason?: string; primaryQualityFailover?: QualityFailoverReceipt; }): ProviderNeutralRouteReceipt { diff --git a/tests/modelAdapterProviderNeutralRoute.test.ts b/tests/modelAdapterProviderNeutralRoute.test.ts index f8b19d87..01debc07 100644 --- a/tests/modelAdapterProviderNeutralRoute.test.ts +++ b/tests/modelAdapterProviderNeutralRoute.test.ts @@ -205,9 +205,34 @@ describe("provider-neutral NodeAgent free-first routing", () => { }); }); - it("does not cross providers for an ordinary candidate-scoped OpenRouter failure", async () => { + it("falls back after the governed free ladder exhausts candidate-scoped failures", async () => { process.env.GOOGLE_GENERATIVE_AI_API_KEY = "test-google-key"; - generateTextMock.mockRejectedValueOnce(new Error("Provider request failed 503: route unavailable")); + generateTextMock + .mockRejectedValueOnce(new Error("Provider request failed 503: route unavailable")) + .mockImplementationOnce(async (options: { model: { id: string } }) => successfulTurn(options.model.id)); + + const { route, step } = await runNeutralRoute(); + + expect(generateTextMock.mock.calls.map((call) => call[0].model)).toEqual([ + { provider: "openrouter", id: OPENROUTER_MODEL }, + { provider: "google", id: GOOGLE_MODEL }, + ]); + expect(route.name).toBe(GOOGLE_MODEL); + expect(step.providerRoute).toMatchObject({ + providerNeutral: { + primary: { + outcome: "free_routes_exhausted", + reason: "provider_free_routes_candidates_exhausted", + qualityFailover: { stopReason: "candidates_exhausted" }, + }, + selected: { route: "google_direct", model: GOOGLE_MODEL }, + }, + }); + }); + + it("does not cross providers for a global authentication boundary", async () => { + process.env.GOOGLE_GENERATIVE_AI_API_KEY = "test-google-key"; + generateTextMock.mockRejectedValueOnce(new Error("Provider request failed 401: Unauthorized")); const error = await runNeutralRoute().then(() => undefined, (failure: unknown) => failure); From 0655606a741804597aafc81d579d49e33936c62d Mon Sep 17 00:00:00 2001 From: homen Date: Wed, 15 Jul 2026 05:39:07 -0700 Subject: [PATCH 09/12] fix(package scripts src tests): preserve SpreadsheetBench workbook packages --- package-lock.json | 1 + package.json | 1 + scripts/spreadsheetbench-refresh-excel.py | 592 +++++++++++- src/eval/spreadsheetBenchNodeAgentBridge.ts | 122 ++- src/eval/spreadsheetBenchScorer.ts | 8 +- src/eval/spreadsheetBenchWorkbookEmitter.ts | 891 ++++++++++++++++++ tests/spreadsheetBenchNodeAgentBridge.test.ts | 72 +- tests/spreadsheetBenchScorer.test.ts | 63 ++ tests/spreadsheetBenchWorkbookEmitter.test.ts | 365 +++++++ tests/spreadsheetbench_refresh_excel_test.py | 292 ++++++ 10 files changed, 2369 insertions(+), 38 deletions(-) create mode 100644 src/eval/spreadsheetBenchWorkbookEmitter.ts create mode 100644 tests/spreadsheetBenchWorkbookEmitter.test.ts create mode 100644 tests/spreadsheetbench_refresh_excel_test.py diff --git a/package-lock.json b/package-lock.json index 7b5f9681..9701fe4f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -56,6 +56,7 @@ "react": "^19.0.0", "react-dom": "^19.0.0", "react-pdf": "^10.4.1", + "saxes": "5.0.1", "streamdown": "^2.5.0", "tailwind-merge": "^3.6.0", "tokenlens": "^1.3.1", diff --git a/package.json b/package.json index 5f70b07c..6bcfcd49 100644 --- a/package.json +++ b/package.json @@ -308,6 +308,7 @@ "react": "^19.0.0", "react-dom": "^19.0.0", "react-pdf": "^10.4.1", + "saxes": "5.0.1", "streamdown": "^2.5.0", "tailwind-merge": "^3.6.0", "tokenlens": "^1.3.1", diff --git a/scripts/spreadsheetbench-refresh-excel.py b/scripts/spreadsheetbench-refresh-excel.py index 457cb6eb..562c355b 100644 --- a/scripts/spreadsheetbench-refresh-excel.py +++ b/scripts/spreadsheetbench-refresh-excel.py @@ -4,17 +4,22 @@ import argparse import hashlib +import html import json +import math import os +import posixpath +import re import shutil import subprocess import tempfile import time +import zipfile +from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path - -import win32com.client - +from typing import Any +from xml.etree import ElementTree def sha256_file(path: Path) -> str: digest = hashlib.sha256() @@ -24,12 +29,63 @@ def sha256_file(path: Path) -> str: return digest.hexdigest() +XML_PREFIX = r'(?:[A-Za-z_][\w.-]*:)?' +CELL_RE = re.compile( + rf'<(?P{XML_PREFIX}c)\b(?P[^>]*?)(?:(?P/>)|>(?P[\s\S]*?))', + re.IGNORECASE, +) +FORMULA_RE = re.compile( + rf'<(?P{XML_PREFIX}f)\b(?P[^>]*?)(?:/>|>(?P[\s\S]*?))', + re.IGNORECASE, +) +VALUE_RE = re.compile( + rf'<(?P{XML_PREFIX}v)\b(?P[^>]*?)(?:(?P/>)|>(?P[\s\S]*?))', + re.IGNORECASE, +) +CELL_REF_RE = re.compile(r'\br\s*=\s*["\']([^"\']+)["\']', re.IGNORECASE) +CELL_TYPE_RE = re.compile(r'\s+t\s*=\s*(["\'])[\s\S]*?\1', re.IGNORECASE) +ADDRESS_RE = re.compile(r'^([A-Z]{1,3})([1-9][0-9]*)$') +RANGE_RE = re.compile(r'^\$?([A-Z]{1,3})\$?([1-9][0-9]*)(?::\$?([A-Z]{1,3})\$?([1-9][0-9]*))?$') +EXCEL_ESCAPE_RE = re.compile(r'_[xX][0-9A-Fa-f]{4}_') +MAX_FORMULA_CACHE_TARGETS_PER_SHEET = 1_000_000 +EXCEL_ERROR_CODES = { + 2000: "#NULL!", + 2007: "#DIV/0!", + 2015: "#VALUE!", + 2023: "#REF!", + 2029: "#NAME?", + 2036: "#NUM!", + 2042: "#N/A", + 2043: "#GETTING_DATA", + 2045: "#SPILL!", + 2046: "#BLOCKED!", + 2047: "#UNKNOWN!", + 2048: "#FIELD!", + 2049: "#DATA!", + 2050: "#CALC!", + 2051: "#CONNECT!", + 2052: "#BUSY!", + 2053: "#PYTHON!", +} + + +@dataclass(frozen=True) +class FormulaCache: + cell_type: str | None + value_xml: str + + def refresh_with_excel(excel, path: Path, record: dict[str, object]) -> None: + formula_cells = workbook_formula_cells(path) + formula_count = sum(len(addresses) for _, addresses in formula_cells.values()) + record["formulaCellCount"] = formula_count + if formula_count == 0: + record["cacheWriteMode"] = "no_formula_caches" + return workbook = None - replacement: Path | None = None open_options = { "UpdateLinks": 0, - "ReadOnly": False, + "ReadOnly": True, "IgnoreReadOnlyRecommended": True, "AddToMru": False, } @@ -42,18 +98,17 @@ def refresh_with_excel(excel, path: Path, record: dict[str, object]) -> None: workbook = excel.Workbooks.Open(str(path), CorruptLoad=1, **open_options) record["openMode"] = "repair" excel.CalculateFullRebuild() - if record["openMode"] == "repair": - replacement = path.with_name(f".{path.stem}.excel-repaired.xlsx") - if replacement.exists(): - replacement.unlink() - workbook.SaveAs(str(replacement), FileFormat=51, CreateBackup=False) - else: - workbook.Save() + values = collect_stable_formula_values(excel, workbook, formula_cells, record) finally: if workbook is not None: workbook.Close(SaveChanges=False) - if replacement is not None: - os.replace(replacement, path) + patch_formula_caches( + path, + formula_cells, + values, + expected_sha256=str(record["beforeSha256"]), + ) + record["cacheWriteMode"] = "original_package_formula_cache_patch" def refresh_with_libreoffice(path: Path) -> str: @@ -88,6 +143,474 @@ def refresh_with_libreoffice(path: Path) -> str: return (result.stdout or result.stderr).strip() +def workbook_formula_cells(path: Path) -> dict[str, tuple[str, list[str]]]: + with zipfile.ZipFile(path, "r") as package: + sheet_parts = workbook_sheet_parts(package) + result: dict[str, tuple[str, list[str]]] = {} + for sheet_name, part_path in sheet_parts.items(): + xml = package.read(part_path).decode("utf-8") + worksheet = ElementTree.fromstring(xml) + addresses: set[str] = set() + for cell in worksheet.iter(): + if local_name(cell.tag) != "c": + continue + address = normalize_address(cell.attrib.get("r", "")) + formula = next((child for child in cell if local_name(child.tag) == "f"), None) + if not address or formula is None: + continue + addresses.add(address) + formula_type = formula.attrib.get("t") + formula_ref = formula.attrib.get("ref") + if formula_ref and formula_type in {"array", "dataTable"}: + addresses.update(expand_range_addresses(formula_ref)) + if addresses: + if len(addresses) > MAX_FORMULA_CACHE_TARGETS_PER_SHEET: + raise RuntimeError( + f"worksheet {sheet_name} declares {len(addresses)} formula cache targets; " + f"limit is {MAX_FORMULA_CACHE_TARGETS_PER_SHEET}" + ) + result[sheet_name] = (part_path, sorted(addresses, key=parse_address)) + return result + + +def workbook_sheet_parts(package: zipfile.ZipFile) -> dict[str, str]: + workbook = ElementTree.fromstring(package.read("xl/workbook.xml")) + relationships = ElementTree.fromstring(package.read("xl/_rels/workbook.xml.rels")) + rels: dict[str, str] = {} + for relationship in relationships: + if local_name(relationship.tag) != "Relationship": + continue + relationship_id = relationship.attrib.get("Id") + target = relationship.attrib.get("Target") + relationship_type = relationship.attrib.get("Type", "") + if not relationship_id or not target: + continue + if relationship.attrib.get("TargetMode", "").lower() == "external": + continue + if not relationship_type.endswith("/worksheet"): + continue + rels[relationship_id] = normalize_workbook_target(target) + result: dict[str, str] = {} + for sheet in workbook.iter(): + if local_name(sheet.tag) != "sheet": + continue + relationship_id = next( + (value for key, value in sheet.attrib.items() if local_name(key) == "id"), + None, + ) + sheet_name = sheet.attrib.get("name") + if not sheet_name or not relationship_id: + raise RuntimeError("workbook contains a worksheet without a name or relationship id") + path = rels.get(relationship_id) + if not path: + raise RuntimeError(f"worksheet {sheet_name} has no internal worksheet relationship") + if path not in package.namelist(): + raise RuntimeError(f"worksheet {sheet_name} relationship target is missing: {path}") + if sheet_name in result: + raise RuntimeError(f"workbook contains duplicate worksheet name {sheet_name}") + result[sheet_name] = path + return result + + +def normalize_workbook_target(target: str) -> str: + normalized = target.replace("\\", "/") + if normalized.startswith("/"): + resolved = posixpath.normpath(normalized).lstrip("/") + else: + resolved = posixpath.normpath(posixpath.join(posixpath.dirname("xl/workbook.xml"), normalized)) + if not resolved or resolved == ".." or resolved.startswith("../"): + raise RuntimeError(f"worksheet relationship escapes the workbook package: {target}") + return resolved + + +def local_name(name: str) -> str: + return name.rsplit("}", 1)[-1] + + +def collect_formula_values( + workbook, + formula_cells: dict[str, tuple[str, list[str]]], +) -> dict[str, dict[str, Any]]: + values: dict[str, dict[str, Any]] = {} + for sheet_name, (_, addresses) in formula_cells.items(): + worksheet = workbook.Worksheets(sheet_name) + sheet_values: dict[str, Any] = {} + for min_row, max_row, min_col, max_col in contiguous_formula_areas(addresses): + block = worksheet.Range( + address_from_position(min_row, min_col), + address_from_position(max_row, max_col), + ).Value2 + matrix = com_range_matrix(block, max_row - min_row + 1, max_col - min_col + 1) + for row in range(min_row, max_row + 1): + for col in range(min_col, max_col + 1): + sheet_values[address_from_position(row, col)] = matrix[row - min_row][col - min_col] + missing = set(addresses).difference(sheet_values) + if missing: + raise RuntimeError(f"Excel omitted {len(missing)} formula cache value(s) on {sheet_name}") + values[sheet_name] = {address: sheet_values[address] for address in addresses} + return values + + +def collect_stable_formula_values( + excel, + workbook, + formula_cells: dict[str, tuple[str, list[str]]], + record: dict[str, object], + timeout_seconds: float = 10.0, +) -> dict[str, dict[str, Any]]: + state = int(excel.CalculationState) + record["calculationStateAfterFullRebuild"] = state + values = collect_formula_values(workbook, formula_cells) + if state == 0: + record["calculationStabilityCheck"] = "excel_done" + return values + + deadline = time.monotonic() + timeout_seconds + stable_reads = 0 + while time.monotonic() < deadline: + time.sleep(0.25) + current = collect_formula_values(workbook, formula_cells) + if formula_value_maps_equal(values, current): + stable_reads += 1 + if stable_reads >= 2: + record["calculationStabilityCheck"] = "stable_while_excel_pending" + record["calculationStateAfterStabilityCheck"] = int(excel.CalculationState) + return current + else: + stable_reads = 0 + values = current + raise RuntimeError( + f"Excel calculation remained unstable for {timeout_seconds:.1f}s " + f"(state={int(excel.CalculationState)})" + ) + + +def formula_value_maps_equal(left: dict[str, dict[str, Any]], right: dict[str, dict[str, Any]]) -> bool: + if left.keys() != right.keys(): + return False + for sheet_name, left_values in left.items(): + right_values = right[sheet_name] + if left_values.keys() != right_values.keys(): + return False + for address, left_value in left_values.items(): + right_value = right_values[address] + if isinstance(left_value, float) and isinstance(right_value, float): + if math.isnan(left_value) and math.isnan(right_value): + continue + if left_value != right_value: + return False + return True + + +def parse_address(address: str) -> tuple[int, int]: + match = ADDRESS_RE.fullmatch(normalize_address(address)) + if not match: + raise ValueError(f"invalid formula cell address {address}") + column = 0 + for char in match.group(1): + column = column * 26 + ord(char) - 64 + return int(match.group(2)), column + + +def normalize_address(address: str) -> str: + return address.replace("$", "").strip().upper() + + +def expand_range_addresses(reference: str) -> list[str]: + match = RANGE_RE.fullmatch(reference.replace(" ", "").upper()) + if not match: + raise RuntimeError(f"invalid formula result range {reference}") + start_row, start_col = parse_address(f"{match.group(1)}{match.group(2)}") + end_row, end_col = parse_address( + f"{match.group(3) or match.group(1)}{match.group(4) or match.group(2)}" + ) + min_row, max_row = sorted((start_row, end_row)) + min_col, max_col = sorted((start_col, end_col)) + area = (max_row - min_row + 1) * (max_col - min_col + 1) + if area > MAX_FORMULA_CACHE_TARGETS_PER_SHEET: + raise RuntimeError( + f"formula result range {reference} contains {area} cells; " + f"limit is {MAX_FORMULA_CACHE_TARGETS_PER_SHEET}" + ) + return [ + address_from_position(row, col) + for row in range(min_row, max_row + 1) + for col in range(min_col, max_col + 1) + ] + + +def contiguous_formula_areas(addresses: list[str]) -> list[tuple[int, int, int, int]]: + columns_by_row: dict[int, set[int]] = {} + for address in addresses: + row, col = parse_address(address) + columns_by_row.setdefault(row, set()).add(col) + + intervals_by_row: dict[int, list[tuple[int, int]]] = {} + for row, columns in columns_by_row.items(): + intervals: list[tuple[int, int]] = [] + start = previous = None + for col in sorted(columns): + if start is None: + start = previous = col + elif col == previous + 1: + previous = col + else: + intervals.append((start, previous)) + start = previous = col + if start is not None and previous is not None: + intervals.append((start, previous)) + intervals_by_row[row] = intervals + + areas: list[tuple[int, int, int, int]] = [] + active: dict[tuple[int, int], tuple[int, int]] = {} + previous_row: int | None = None + for row in sorted(intervals_by_row): + current = set(intervals_by_row[row]) + if previous_row is None or row != previous_row + 1: + for (min_col, max_col), (min_row, max_row) in active.items(): + areas.append((min_row, max_row, min_col, max_col)) + active.clear() + for interval in list(active): + if interval not in current: + min_row, max_row = active.pop(interval) + areas.append((min_row, max_row, interval[0], interval[1])) + for interval in current: + if interval in active: + min_row, _ = active[interval] + active[interval] = (min_row, row) + else: + active[interval] = (row, row) + previous_row = row + for (min_col, max_col), (min_row, max_row) in active.items(): + areas.append((min_row, max_row, min_col, max_col)) + return sorted(areas) + + +def address_from_position(row: int, col: int) -> str: + letters = "" + remaining = col + while remaining > 0: + remaining, index = divmod(remaining - 1, 26) + letters = chr(65 + index) + letters + return f"{letters}{row}" + + +def com_range_matrix(value: Any, rows: int, cols: int) -> list[list[Any]]: + if rows == 1 and cols == 1: + return [[value]] + if not isinstance(value, tuple): + return [[value for _ in range(cols)] for _ in range(rows)] + if rows == 1 and value and not isinstance(value[0], tuple): + return [list(value)] + if cols == 1 and value and not isinstance(value[0], tuple): + return [[item] for item in value] + return [list(row) for row in value] + + +def patch_formula_caches( + path: Path, + formula_cells: dict[str, tuple[str, list[str]]], + values: dict[str, dict[str, Any]], + *, + expected_sha256: str, +) -> None: + part_values = { + part_path: values[sheet_name] + for sheet_name, (part_path, _) in formula_cells.items() + } + descriptor, replacement_name = tempfile.mkstemp( + prefix=f".{path.name}.", + suffix=".cache-patched", + dir=path.parent, + ) + os.close(descriptor) + replacement = Path(replacement_name) + try: + if sha256_file(path) != expected_sha256: + raise RuntimeError(f"workbook changed before cache patching: {path}") + with zipfile.ZipFile(path, "r") as source, zipfile.ZipFile(replacement, "w") as output: + source_names = source.namelist() + output.comment = source.comment + for info in source.infolist(): + data = source.read(info.filename) + caches = part_values.get(info.filename) + if caches: + xml = data.decode("utf-8") + patched_xml, patched_addresses = patch_worksheet_formula_caches(xml, caches) + expected_addresses = set(caches) + if patched_addresses != expected_addresses: + missing = expected_addresses.difference(patched_addresses) + raise RuntimeError( + f"worksheet {info.filename} omitted {len(missing)} formula cache target(s)" + ) + ElementTree.fromstring(patched_xml) + data = patched_xml.encode("utf-8") + output.writestr(info, data) + with zipfile.ZipFile(replacement, "r") as candidate: + if candidate.namelist() != source_names: + raise RuntimeError("cache patch changed workbook package entries") + bad_entry = candidate.testzip() + if bad_entry: + raise RuntimeError(f"cache-patched workbook has an invalid ZIP entry: {bad_entry}") + if sha256_file(path) != expected_sha256: + raise RuntimeError(f"workbook changed concurrently during cache patching: {path}") + os.replace(replacement, path) + finally: + if replacement.exists(): + replacement.unlink() + + +def patch_worksheet_formula_caches(xml: str, values: dict[str, Any]) -> tuple[str, set[str]]: + patched_addresses: set[str] = set() + + def replace_cell(match: re.Match[str]) -> str: + attrs = match.group("attrs") + body = match.group("body") or "" + cell_ref = CELL_REF_RE.search(attrs) + address = normalize_address(cell_ref.group(1)) if cell_ref else "" + if address not in values: + return match.group(0) + cache = values[address] if isinstance(values[address], FormulaCache) else serialize_formula_cache(values[address]) + next_attrs = CELL_TYPE_RE.sub("", attrs) + if cache.cell_type: + next_attrs = f'{next_attrs} t="{cache.cell_type}"' + + cell_tag = match.group("tag") + value_tag = f'{cell_tag.rsplit(":", 1)[0]}:v' if ":" in cell_tag else "v" + + def replace_value(value_match: re.Match[str]) -> str: + return ( + f'<{value_match.group("tag")}{value_match.group("attrs")}>' + f'{cache.value_xml}' + ) + + cached_value = f"<{value_tag}>{cache.value_xml}" + next_body = VALUE_RE.sub(replace_value, body, count=1) + if next_body == body: + formula_match = FORMULA_RE.search(body) + if formula_match: + next_body = f"{body[:formula_match.end()]}{cached_value}{body[formula_match.end():]}" + else: + next_body = f"{body}{cached_value}" + patched_addresses.add(address) + return f"<{cell_tag}{next_attrs}>{next_body}" + + return CELL_RE.sub(replace_cell, xml), patched_addresses + + +def serialize_formula_cache(value: Any) -> FormulaCache: + if value is None: + return FormulaCache(None, "") + if isinstance(value, bool): + return FormulaCache("b", "1" if value else "0") + if isinstance(value, int): + error = EXCEL_ERROR_CODES.get(value & 0xFFFF) + if error: + return FormulaCache("e", html.escape(error, quote=False)) + if isinstance(value, (int, float)): + if not math.isfinite(value): + raise RuntimeError(f"Excel returned a non-finite formula cache value: {value}") + return FormulaCache(None, repr(value)) + return FormulaCache("str", encode_spreadsheetml_string(str(value))) + + +def encode_spreadsheetml_string(value: str) -> str: + escaped_literals = EXCEL_ESCAPE_RE.sub(lambda match: f"_x005F_{match.group(0)[1:]}", value) + encoded = "".join( + f"_x{ord(char):04X}_" + if ord(char) == 0x0D or ord(char) < 0x20 and char not in {"\t", "\n"} + else char + for char in escaped_literals + ) + return html.escape(encoded, quote=False) + + +def refresh_with_libreoffice_cache_patch(path: Path, record: dict[str, object]) -> None: + formula_cells = workbook_formula_cells(path) + formula_count = sum(len(addresses) for _, addresses in formula_cells.values()) + record["formulaCellCount"] = formula_count + if formula_count == 0: + record["cacheWriteMode"] = "no_formula_caches" + return + with tempfile.TemporaryDirectory(prefix="ssb-v2-cache-fallback-") as temp_dir: + temporary = Path(temp_dir) / path.name + shutil.copy2(path, temporary) + record["libreOfficeOutput"] = refresh_with_libreoffice(temporary) + values = read_formula_caches(temporary, formula_cells) + patch_formula_caches( + path, + formula_cells, + values, + expected_sha256=str(record["beforeSha256"]), + ) + record["cacheWriteMode"] = "original_package_formula_cache_patch" + + +def read_formula_caches( + path: Path, + source_formula_cells: dict[str, tuple[str, list[str]]], +) -> dict[str, dict[str, FormulaCache]]: + with zipfile.ZipFile(path, "r") as package: + refreshed_parts = workbook_sheet_parts(package) + values: dict[str, dict[str, FormulaCache]] = {} + for sheet_name, (_, addresses) in source_formula_cells.items(): + refreshed_path = refreshed_parts.get(sheet_name) + if not refreshed_path: + raise RuntimeError(f"refreshed workbook is missing worksheet {sheet_name}") + xml = package.read(refreshed_path).decode("utf-8") + by_address: dict[str, FormulaCache] = {} + requested = set(addresses) + for match in CELL_RE.finditer(xml): + cell_ref = CELL_REF_RE.search(match.group("attrs")) + address = cell_ref.group(1).replace("$", "").upper() if cell_ref else "" + if address not in requested: + continue + body = match.group("body") or "" + value_match = VALUE_RE.search(body) + raw = value_match.group("body") or "" if value_match else "" + cell_type = next( + (item.group(2) for item in re.finditer(r'\bt\s*=\s*(["\'])([^"\']+)\1', match.group("attrs"), re.IGNORECASE)), + None, + ) + if cell_type == "inlineStr": + text = "".join( + html.unescape(text_match.group(1)) + for text_match in re.finditer( + rf'<{XML_PREFIX}t\b[^>]*>([\s\S]*?)', + body, + re.IGNORECASE, + ) + ) + by_address[address] = FormulaCache("str", encode_spreadsheetml_string(text)) + else: + by_address[address] = FormulaCache(cell_type, raw) + missing = requested.difference(by_address) + if missing: + raise RuntimeError(f"refreshed workbook omitted {len(missing)} formula cache cell(s) on {sheet_name}") + values[sheet_name] = by_address + return values + + +def create_excel_application(): + import win32com.client # Imported lazily so OOXML helpers remain cross-platform testable. + + excel = win32com.client.DispatchEx("Excel.Application") + try: + excel.Visible = False + excel.DisplayAlerts = False + excel.AskToUpdateLinks = False + excel.EnableEvents = False + excel.AutomationSecurity = 3 # msoAutomationSecurityForceDisable + version = str(excel.Version) + return excel, version + except Exception: + try: + excel.Quit() + except Exception: + pass + raise + + def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--dir", required=True, help="Root containing *_output.xlsx files") @@ -110,14 +633,15 @@ def main() -> int: started = time.time() records: list[dict[str, object]] = [] - excel = win32com.client.DispatchEx("Excel.Application") - excel.Visible = False - excel.DisplayAlerts = False - excel.AskToUpdateLinks = False - excel.EnableEvents = False - excel.AutomationSecurity = 3 # msoAutomationSecurityForceDisable - version = str(excel.Version) + excel = None + version = "unavailable" + excel_initialization_error: str | None = None + excel_quit_error: str | None = None try: + try: + excel, version = create_excel_application() + except Exception as exc: + excel_initialization_error = str(exc) for index, path in enumerate(files, start=1): before = sha256_file(path) record: dict[str, object] = { @@ -125,12 +649,17 @@ def main() -> int: "beforeSha256": before, } try: - try: - refresh_with_excel(excel, path, record) - except Exception as excel_error: - record["excelError"] = str(excel_error) + if excel is not None: + try: + refresh_with_excel(excel, path, record) + except Exception as excel_error: + record["excelError"] = str(excel_error) + record["openMode"] = "libreoffice_fallback" + refresh_with_libreoffice_cache_patch(path, record) + else: + record["excelInitializationError"] = excel_initialization_error or "Excel initialization failed" record["openMode"] = "libreoffice_fallback" - record["libreOfficeOutput"] = refresh_with_libreoffice(path) + refresh_with_libreoffice_cache_patch(path, record) record["status"] = "refreshed" except Exception as exc: # Refresh errors must remain visible in the receipt. record["status"] = "failed" @@ -140,7 +669,11 @@ def main() -> int: if index % 20 == 0 or index == len(files): print(f"refreshed {index}/{len(files)}", flush=True) finally: - excel.Quit() + if excel is not None: + try: + excel.Quit() + except Exception as exc: + excel_quit_error = str(exc) if previous_records: for record in records: @@ -152,12 +685,15 @@ def main() -> int: receipt = { "schema": 1, "generatedAt": datetime.now(timezone.utc).isoformat(), - "engine": "Microsoft Excel COM with isolated LibreOffice conversion fallback", + "engine": "Microsoft Excel COM cache extraction with isolated LibreOffice cache fallback", "engineVersion": version, + **({"excelInitializationError": excel_initialization_error} if excel_initialization_error else {}), + **({"excelQuitError": excel_quit_error} if excel_quit_error else {}), "policy": { "macros": "disabled", "externalLinkUpdates": "disabled", "calculation": "CalculateFullRebuild", + "persistence": "patch cached formula values into the original OOXML package without saving Excel's formula rewrites", }, "root": root.relative_to(Path.cwd()).as_posix(), "workbookCount": len(records), diff --git a/src/eval/spreadsheetBenchNodeAgentBridge.ts b/src/eval/spreadsheetBenchNodeAgentBridge.ts index b621bfa4..aa0102bf 100644 --- a/src/eval/spreadsheetBenchNodeAgentBridge.ts +++ b/src/eval/spreadsheetBenchNodeAgentBridge.ts @@ -4,6 +4,10 @@ import { basename, dirname, isAbsolute, relative, resolve } from "node:path"; import ExcelJS from "exceljs"; import type { SpreadsheetBenchTrack } from "./spreadsheetBenchAdapter"; import { readSpreadsheetBenchWorkbookForMutation } from "./spreadsheetBenchScorer"; +import { + emitSpreadsheetBenchWorkbookCandidate, + type SpreadsheetBenchWorkbookCellPatch, +} from "./spreadsheetBenchWorkbookEmitter"; import { evaluateFormula, FormulaEvalError, @@ -254,7 +258,11 @@ export async function runSpreadsheetBenchNodeAgentBridge( const recalculation = room.recalculateChangedFormulas(); mkdirSync(dirname(candidateWorkbookPath), { recursive: true }); - await workbook.xlsx.writeFile(candidateWorkbookPath); + await emitSpreadsheetBenchWorkbookCandidate({ + sourceWorkbookPath: task.sourceWorkbookPath, + candidateWorkbookPath, + patches: room.packageMutations(), + }); const candidateWorkbookSha256 = sha256File(candidateWorkbookPath); const stages = buildStageReceipts(traceId, frameReceipt); const mutatingTask = room.taskInspection().mutatingTask; @@ -324,7 +332,12 @@ class SpreadsheetBenchWorkbookRoomTools implements RoomTools { private readonly scanMaxCells: number; private readonly versions = new Map(); private readonly locks = new Map(); - private readonly changedTargets = new Map(); + private readonly changedTargets = new Map(); private readonly chat: string[] = []; private lockCounter = 0; private draftCounter = 0; @@ -353,6 +366,20 @@ class SpreadsheetBenchWorkbookRoomTools implements RoomTools { return this.mutations; } + packageMutations(): SpreadsheetBenchWorkbookCellPatch[] { + return [...this.changedTargets.values()] + .sort((left, right) => left.sheet.localeCompare(right.sheet) || left.address.localeCompare(right.address)) + .flatMap((target) => { + const cell = this.sheet(target.sheet).getCell(target.address); + if (sameWorkbookCellSemanticState( + target.originalState, + workbookCellSemanticState(cell), + target.numFmtTouched, + )) return []; + return [workbookCellPatch(target.sheet, target.address, cell, target.numFmtTouched)]; + }); + } + taskInspection() { this.recalculateChangedFormulas(); return inspectWorkbookTask({ @@ -524,6 +551,7 @@ class SpreadsheetBenchWorkbookRoomTools implements RoomTools { ): Promise { const target = this.target(elementId, this.sheet(artifactId).name); const cell = target.sheet.getCell(target.address); + const originalState = workbookCellSemanticState(cell); const currentVersion = this.version(target.sheet.name, target.address, cell); if (kind === "create" && currentVersion !== 0) { return { ok: false, conflict: true, expected: baseVersion, actual: currentVersion }; @@ -539,9 +567,13 @@ class SpreadsheetBenchWorkbookRoomTools implements RoomTools { } const nextVersion = currentVersion + 1; this.versions.set(cellKey(target.sheet.name, target.address), nextVersion); - this.changedTargets.set(cellKey(target.sheet.name, target.address), { + const targetKey = cellKey(target.sheet.name, target.address); + const priorTarget = this.changedTargets.get(targetKey); + this.changedTargets.set(targetKey, { sheet: target.sheet.name, address: target.address, + numFmtTouched: priorTarget?.numFmtTouched === true || typeof asRecord(value)?.numFmt === "string", + originalState: priorTarget?.originalState ?? originalState, }); this.workbookVersion += 1; this.mutations += 1; @@ -1860,6 +1892,83 @@ function roomCellScalar(cell: ExcelJS.Cell): unknown { return value; } +type WorkbookCellSemanticState = { + kind: "clear" | "formula" | "value"; + formula?: string; + valueKey?: string; + numFmt: string; +}; + +function workbookCellSemanticState(cell: ExcelJS.Cell): WorkbookCellSemanticState { + const formula = cellFormula(cell); + if (formula) return { kind: "formula", formula, numFmt: cell.numFmt }; + if (cell.value === null || cell.value === undefined) return { kind: "clear", numFmt: cell.numFmt }; + return { kind: "value", valueKey: stableWorkbookValue(cell.value), numFmt: cell.numFmt }; +} + +function sameWorkbookCellSemanticState( + left: WorkbookCellSemanticState, + right: WorkbookCellSemanticState, + compareNumberFormat: boolean, +): boolean { + return left.kind === right.kind + && left.formula === right.formula + && left.valueKey === right.valueKey + && (!compareNumberFormat || left.numFmt === right.numFmt); +} + +function stableWorkbookValue(value: unknown): string { + if (value === undefined) return "undefined"; + if (value === null) return "null"; + if (value instanceof Date) return `date:${Number.isFinite(value.getTime()) ? value.getTime() : "invalid"}`; + if (Array.isArray(value)) return `[${value.map(stableWorkbookValue).join(",")}]`; + if (value && typeof value === "object") { + return `{${Object.entries(value as Record) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, nested]) => `${JSON.stringify(key)}:${stableWorkbookValue(nested)}`) + .join(",")}}`; + } + return `${typeof value}:${String(value)}`; +} + +function workbookCellPatch( + sheet: string, + address: string, + cell: ExcelJS.Cell, + numFmtTouched: boolean, +): SpreadsheetBenchWorkbookCellPatch { + const formula = cellFormula(cell); + const valueRecord = asRecord(cell.value); + const numFmt = numFmtTouched ? cell.numFmt : undefined; + if (formula) { + const hasCachedResult = Boolean(valueRecord && Object.prototype.hasOwnProperty.call(valueRecord, "result")); + return { + sheet, + address, + kind: "formula", + formula, + hasCachedResult, + ...(hasCachedResult ? { cachedResult: valueRecord?.result } : {}), + ...(numFmt === undefined ? {} : { numFmt }), + }; + } + if (cell.value === null || cell.value === undefined) { + return { + sheet, + address, + kind: "clear", + ...(numFmt === undefined ? {} : { numFmt }), + }; + } + return { + sheet, + address, + kind: "value", + value: cell.value, + ...(numFmt === undefined ? {} : { numFmt }), + }; +} + function formulaCachedScalar(cell: ExcelJS.Cell): FormulaEngineCellValue | undefined { const value = cell.value; if (!value || typeof value !== "object" || !("result" in value)) return undefined; @@ -1874,7 +1983,10 @@ function formulaEnginePrimitive(value: unknown): FormulaEngineCellValue | undefi if (value === null || value === undefined) return null; if (typeof value === "number") return Number.isFinite(value) ? value : undefined; if (typeof value === "string" || typeof value === "boolean") return value; - if (value instanceof Date) return (value.getTime() - Date.UTC(1899, 11, 30)) / 86_400_000; + if (value instanceof Date) { + const time = value.getTime(); + return Number.isFinite(time) ? (time - Date.UTC(1899, 11, 30)) / 86_400_000 : undefined; + } return undefined; } @@ -2009,7 +2121,7 @@ function cellKey(sheet: string, address: string): string { function displayValue(value: unknown): string { if (value === null || value === undefined) return ""; - if (value instanceof Date) return value.toISOString(); + if (value instanceof Date) return Number.isFinite(value.getTime()) ? value.toISOString() : ""; if (typeof value === "object") return JSON.stringify(value); return String(value); } diff --git a/src/eval/spreadsheetBenchScorer.ts b/src/eval/spreadsheetBenchScorer.ts index 8f87d7f0..cb182e01 100644 --- a/src/eval/spreadsheetBenchScorer.ts +++ b/src/eval/spreadsheetBenchScorer.ts @@ -329,7 +329,7 @@ function normalizeWorkbookRelationshipTarget(target: string): string { function parseZipWorksheetCells(xml: string, sharedStrings: string[]): Map { const cells = new Map(); - for (const match of xml.matchAll(/]*)(?:\/>|>([\s\S]*?)<\/c>)/gi)) { + for (const match of xml.matchAll(/]*?)(?:\/>|>([\s\S]*?)<\/c>)/gi)) { const attrs = xmlAttrs(match[1]); const address = attrs.r?.replace(/\$/g, "").toUpperCase(); if (!address) continue; @@ -475,6 +475,12 @@ function xmlAttrs(value: string): Record { function decodeXml(value: string): string { return value + .replace(/&#(?:x([0-9a-f]+)|([0-9]+));/gi, (entity, hex: string | undefined, decimal: string | undefined) => { + const codePoint = Number.parseInt(hex ?? decimal ?? "", hex ? 16 : 10); + return Number.isInteger(codePoint) && codePoint >= 0 && codePoint <= 0x10ffff + ? String.fromCodePoint(codePoint) + : entity; + }) .replace(/"/g, "\"") .replace(/'/g, "'") .replace(/</g, "<") diff --git a/src/eval/spreadsheetBenchWorkbookEmitter.ts b/src/eval/spreadsheetBenchWorkbookEmitter.ts new file mode 100644 index 00000000..ca726633 --- /dev/null +++ b/src/eval/spreadsheetBenchWorkbookEmitter.ts @@ -0,0 +1,891 @@ +import { randomUUID } from "node:crypto"; +import { copyFileSync, readFileSync, renameSync, unlinkSync, writeFileSync } from "node:fs"; +import { basename, dirname, join, posix, resolve } from "node:path"; +import JSZip from "jszip"; +import { SaxesParser, type SaxesTagNS } from "saxes"; + +export type SpreadsheetBenchWorkbookCellPatch = { + sheet: string; + address: string; + kind: "clear" | "formula" | "value"; + formula?: string; + hasCachedResult?: boolean; + cachedResult?: unknown; + value?: unknown; + numFmt?: string; +}; + +type WorkbookSheetPart = { + name: string; + path: string; +}; + +type XmlNodeSpan = { + name: string; + local: string; + attrs: Record; + start: number; + openEnd: number; + closeStart: number; + end: number; + selfClosing: boolean; + parent?: XmlNodeSpan; +}; + +type XmlSplice = { + start: number; + end: number; + text: string; +}; + +type CellCoordinate = { + row: number; + col: number; +}; + +type ColumnStyleRange = { + min: number; + max: number; + style: number; +}; + +const BUILTIN_NUMBER_FORMATS = new Map([ + [0, "General"], + [1, "0"], + [2, "0.00"], + [3, "#,##0"], + [4, "#,##0.00"], + [9, "0%"], + [10, "0.00%"], + [11, "0.00E+00"], + [12, "# ?/?"], + [13, "# ??/??"], + [14, "mm-dd-yy"], + [15, "d-mmm-yy"], + [16, "d-mmm"], + [17, "mmm-yy"], + [18, "h:mm AM/PM"], + [19, "h:mm:ss AM/PM"], + [20, "h:mm"], + [21, "h:mm:ss"], + [22, "m/d/yy h:mm"], + [37, "#,##0 ;(#,##0)"], + [38, "#,##0 ;[Red](#,##0)"], + [39, "#,##0.00;(#,##0.00)"], + [40, "#,##0.00;[Red](#,##0.00)"], + [45, "mm:ss"], + [46, "[h]:mm:ss"], + [47, "mmss.0"], + [48, "##0.0E+0"], + [49, "@"], +]); + +export async function emitSpreadsheetBenchWorkbookCandidate(args: { + sourceWorkbookPath: string; + candidateWorkbookPath: string; + patches: SpreadsheetBenchWorkbookCellPatch[]; +}): Promise { + if (resolve(args.sourceWorkbookPath) === resolve(args.candidateWorkbookPath)) { + throw new Error("Workbook candidate path must not overwrite the source workbook"); + } + if (args.patches.length === 0) { + atomicCopyFile(args.sourceWorkbookPath, args.candidateWorkbookPath); + return; + } + + const zip = await JSZip.loadAsync(readFileSync(args.sourceWorkbookPath)); + const sheets = await workbookSheetParts(zip); + const sheetsByName = new Map(sheets.map((sheet) => [sheet.name.toLowerCase(), sheet])); + const patchesBySheet = new Map(); + const finalPatches = new Map(); + for (const patch of args.patches) { + const address = normalizeAddress(patch.address); + if (!parseAddress(address)) throw new Error(`Invalid workbook cell patch address ${patch.sheet}!${patch.address}`); + const key = patch.sheet.toLowerCase(); + finalPatches.set(`${key}\u0000${address}`, { ...patch, address }); + } + for (const patch of finalPatches.values()) { + const key = patch.sheet.toLowerCase(); + const existing = patchesBySheet.get(key) ?? []; + existing.push(patch); + patchesBySheet.set(key, existing); + } + + const stylesPath = "xl/styles.xml"; + const stylesFile = zip.file(stylesPath); + if (!stylesFile) throw new Error(`Workbook package is missing ${stylesPath}`); + const sourceStylesXml = await stylesFile.async("string"); + const styles = new WorkbookStylesEditor(sourceStylesXml); + + for (const [sheetName, patches] of patchesBySheet) { + const sheet = sheetsByName.get(sheetName); + if (!sheet) throw new Error(`Workbook package has no worksheet named ${JSON.stringify(patches[0]?.sheet)}`); + const worksheet = zip.file(sheet.path); + if (!worksheet) throw new Error(`Workbook package is missing ${sheet.path}`); + const patchedXml = patchWorksheetCells(await worksheet.async("string"), patches, styles); + assertWellFormedXml(patchedXml, sheet.path); + zip.file(sheet.path, patchedXml); + } + + const stylesXml = styles.finish(); + assertWellFormedXml(stylesXml, stylesPath); + if (stylesXml !== sourceStylesXml) zip.file(stylesPath, stylesXml); + await markWorkbookForFullCalculation(zip); + + atomicWriteFile( + args.candidateWorkbookPath, + await zip.generateAsync({ type: "nodebuffer", compression: "DEFLATE" }), + ); +} + +class WorkbookStylesEditor { + private readonly sourceXml: string; + private readonly sourceXfs: string[]; + private readonly formatCodeById = new Map(BUILTIN_NUMBER_FORMATS); + private readonly formatIdByCode = new Map(); + private readonly pendingNumberFormats: Array<{ id: number; code: string }> = []; + private readonly pendingXfs: string[] = []; + private readonly styleCache = new Map(); + private nextNumberFormatId: number; + + constructor(xml: string) { + this.sourceXml = xml; + const nodes = indexXmlSpans(xml); + const cellXfs = nodes.find((node) => node.local === "cellXfs"); + if (!cellXfs) throw new Error("Workbook styles XML is missing cellXfs"); + this.sourceXfs = nodes + .filter((node) => node.local === "xf" && node.parent === cellXfs) + .map((node) => xml.slice(node.start, node.end)); + if (this.sourceXfs.length === 0) throw new Error("Workbook styles XML has no cellXfs entries"); + + for (const node of nodes.filter((candidate) => candidate.local === "numFmt")) { + const id = Number(node.attrs.numFmtId); + const code = node.attrs.formatCode; + if (Number.isInteger(id) && code !== undefined) this.formatCodeById.set(id, code); + } + for (const [id, code] of this.formatCodeById) { + if (!this.formatIdByCode.has(code)) this.formatIdByCode.set(code, id); + } + this.nextNumberFormatId = Math.max(164, ...this.formatCodeById.keys()) + 1; + } + + ensureNumberFormatStyle(sourceStyleIndex: number, formatCode: string): number { + const sourceXf = this.sourceXfs[sourceStyleIndex]; + if (!sourceXf) throw new Error(`Workbook cell references missing style index ${sourceStyleIndex}`); + const sourceXfNode = indexXmlSpans(sourceXf).find((node) => node.local === "xf"); + if (!sourceXfNode) throw new Error(`Workbook style ${sourceStyleIndex} is malformed`); + const sourceFormatId = Number(sourceXfNode.attrs.numFmtId ?? 0); + if (this.formatCodeById.get(sourceFormatId) === formatCode) return sourceStyleIndex; + + const cacheKey = `${sourceStyleIndex}\u0000${formatCode}`; + const cached = this.styleCache.get(cacheKey); + if (cached !== undefined) return cached; + const formatId = this.ensureNumberFormat(formatCode); + let nextXf = setElementAttribute(sourceXf, "numFmtId", String(formatId)); + nextXf = setElementAttribute(nextXf, "applyNumberFormat", formatId === 0 ? "0" : "1"); + const styleIndex = this.sourceXfs.length + this.pendingXfs.length; + this.pendingXfs.push(nextXf); + this.styleCache.set(cacheKey, styleIndex); + return styleIndex; + } + + finish(): string { + let xml = this.sourceXml; + if (this.pendingNumberFormats.length > 0) { + const fragments = this.pendingNumberFormats + .map((format) => ``) + .join(""); + let nodes = indexXmlSpans(xml); + const numFmts = nodes.find((node) => node.local === "numFmts"); + if (numFmts) { + const existingCount = nodes.filter((node) => node.local === "numFmt" && node.parent === numFmts).length; + if (numFmts.selfClosing) { + const opening = xml.slice(numFmts.start, numFmts.openEnd).replace(/\s*\/>$/, ">"); + const expanded = `${setXmlAttribute(opening, "count", String(existingCount + this.pendingNumberFormats.length))}${fragments}`; + xml = applyXmlSplices(xml, [{ start: numFmts.start, end: numFmts.end, text: expanded }]); + } else { + const opening = setXmlAttribute(xml.slice(numFmts.start, numFmts.openEnd), "count", String(existingCount + this.pendingNumberFormats.length)); + xml = applyXmlSplices(xml, [ + { start: numFmts.start, end: numFmts.openEnd, text: opening }, + { start: numFmts.closeStart, end: numFmts.closeStart, text: fragments }, + ]); + } + } else { + nodes = indexXmlSpans(xml); + const fonts = nodes.find((node) => node.local === "fonts"); + if (!fonts) throw new Error("Workbook styles XML is missing fonts"); + xml = applyXmlSplices(xml, [{ + start: fonts.start, + end: fonts.start, + text: `${fragments}`, + }]); + } + } + + if (this.pendingXfs.length > 0) { + const nodes = indexXmlSpans(xml); + const cellXfs = nodes.find((node) => node.local === "cellXfs"); + if (!cellXfs || cellXfs.selfClosing) throw new Error("Workbook styles XML has no writable cellXfs collection"); + const count = nodes.filter((node) => node.local === "xf" && node.parent === cellXfs).length; + const opening = setXmlAttribute(xml.slice(cellXfs.start, cellXfs.openEnd), "count", String(count + this.pendingXfs.length)); + xml = applyXmlSplices(xml, [ + { start: cellXfs.start, end: cellXfs.openEnd, text: opening }, + { start: cellXfs.closeStart, end: cellXfs.closeStart, text: this.pendingXfs.join("") }, + ]); + } + return xml; + } + + private ensureNumberFormat(formatCode: string): number { + const existing = this.formatIdByCode.get(formatCode); + if (existing !== undefined) return existing; + const id = this.nextNumberFormatId++; + this.pendingNumberFormats.push({ id, code: formatCode }); + this.formatIdByCode.set(formatCode, id); + this.formatCodeById.set(id, formatCode); + return id; + } +} + +function patchWorksheetCells( + xml: string, + patches: SpreadsheetBenchWorkbookCellPatch[], + styles: WorkbookStylesEditor, +): string { + const nodes = indexXmlSpans(xml); + const sheetData = nodes.find((node) => node.local === "sheetData"); + if (!sheetData) throw new Error("Workbook worksheet XML is missing sheetData"); + const rows = nodes.filter((node) => node.local === "row" && node.parent === sheetData); + const rowsByNumber = new Map(rows.map((row) => [Number(row.attrs.r), row])); + const cells = nodes.filter((node) => node.local === "c" && node.parent?.local === "row"); + const cellsByAddress = new Map(cells.map((cell) => [normalizeAddress(cell.attrs.r ?? ""), cell])); + const columnStyles = worksheetColumnStyles(nodes); + const formulaGroupSplices = prepareFormulaGroupSplices(xml, nodes, patches); + const rowPatches = new Map(); + for (const patch of patches) { + const coordinate = parseAddress(patch.address); + if (!coordinate) throw new Error(`Invalid workbook cell patch address ${patch.sheet}!${patch.address}`); + const existing = rowPatches.get(coordinate.row) ?? []; + existing.push(patch); + rowPatches.set(coordinate.row, existing); + } + + if (sheetData.selfClosing && rows.length === 0) { + const content = [...rowPatches.entries()] + .sort(([left], [right]) => left - right) + .map(([rowNumber, rowItems]) => { + const serialized = rowItems + .sort((left, right) => parseAddress(left.address)!.col - parseAddress(right.address)!.col) + .map((patch) => serializeCellPatch( + xml, + patch, + undefined, + effectiveCellStyle(undefined, undefined, parseAddress(patch.address)!.col, columnStyles), + styles, + )) + .join(""); + return `${serialized}`; + }) + .join(""); + const opening = xml.slice(sheetData.start, sheetData.openEnd).replace(/\s*\/>$/, ">"); + return updateWorksheetDimension( + applyXmlSplices(xml, [{ + start: sheetData.start, + end: sheetData.end, + text: `${opening}${content}`, + }]), + ); + } + + const splices: XmlSplice[] = [...formulaGroupSplices]; + const insertions = new Map>(); + for (const [rowNumber, items] of rowPatches) { + const row = rowsByNumber.get(rowNumber); + if (!row) { + const text = items + .sort((left, right) => parseAddress(left.address)!.col - parseAddress(right.address)!.col) + .map((patch) => serializeCellPatch( + xml, + patch, + undefined, + effectiveCellStyle(undefined, undefined, parseAddress(patch.address)!.col, columnStyles), + styles, + )) + .join(""); + const nextRow = rows.find((candidate) => Number(candidate.attrs.r) > rowNumber); + const position = nextRow?.start ?? sheetData.closeStart; + const pending = insertions.get(position) ?? []; + pending.push({ col: rowNumber, text: `${text}` }); + insertions.set(position, pending); + continue; + } + + if (row.selfClosing) { + const text = items + .sort((left, right) => parseAddress(left.address)!.col - parseAddress(right.address)!.col) + .map((patch) => serializeCellPatch( + xml, + patch, + undefined, + effectiveCellStyle(undefined, row, parseAddress(patch.address)!.col, columnStyles), + styles, + )) + .join(""); + const opening = xml.slice(row.start, row.openEnd).replace(/\s*\/>$/, ">"); + splices.push({ start: row.start, end: row.end, text: `${opening}${text}` }); + continue; + } + + const rowCells = cells + .filter((cell) => cell.parent === row) + .sort((left, right) => parseAddress(left.attrs.r ?? "")!.col - parseAddress(right.attrs.r ?? "")!.col); + for (const patch of items) { + const coordinate = parseAddress(patch.address)!; + const sourceCell = cellsByAddress.get(patch.address); + if (sourceCell) { + const sourceStyle = effectiveCellStyle(sourceCell, row, coordinate.col, columnStyles); + splices.push({ + start: sourceCell.start, + end: sourceCell.end, + text: serializeCellPatch(xml, patch, sourceCell, sourceStyle, styles), + }); + continue; + } + const nextCell = rowCells.find((cell) => parseAddress(cell.attrs.r ?? "")!.col > coordinate.col); + const position = nextCell?.start ?? row.closeStart; + const pending = insertions.get(position) ?? []; + pending.push({ + col: coordinate.col, + text: serializeCellPatch( + xml, + patch, + undefined, + effectiveCellStyle(undefined, row, coordinate.col, columnStyles), + styles, + ), + }); + insertions.set(position, pending); + } + } + + for (const [position, values] of insertions) { + splices.push({ + start: position, + end: position, + text: values.sort((left, right) => left.col - right.col).map((value) => value.text).join(""), + }); + } + return updateWorksheetDimension(applyXmlSplices(xml, splices)); +} + +function prepareFormulaGroupSplices( + xml: string, + nodes: XmlNodeSpan[], + patches: SpreadsheetBenchWorkbookCellPatch[], +): XmlSplice[] { + const patchAddresses = new Set(patches.map((patch) => patch.address)); + const formulaNodes = nodes.filter((node) => node.local === "f" && node.parent?.local === "c"); + + for (const formula of formulaNodes.filter((node) => node.attrs.t === "array" || node.attrs.t === "dataTable")) { + const masterAddress = normalizeAddress(formula.parent?.attrs.r ?? ""); + const range = parseCellRange(formula.attrs.ref ?? masterAddress); + if (!range) { + if (patchAddresses.has(masterAddress)) { + throw new Error(`Refusing to patch malformed ${formula.attrs.t} formula group at ${patches[0]?.sheet}!${masterAddress}`); + } + continue; + } + const covered = [...patchAddresses].filter((address) => coordinateInRange(parseAddress(address), range)).length; + if (covered > 0 && covered !== range.area) { + throw new Error( + `Refusing to patch ${formula.attrs.t} formula range ${patches[0]?.sheet}!${range.ref} without all ${range.area} cells`, + ); + } + } + + const sharedGroups = new Map>(); + for (const formula of formulaNodes.filter((node) => node.attrs.t === "shared")) { + const si = formula.attrs.si; + const cell = formula.parent; + const address = normalizeAddress(cell?.attrs.r ?? ""); + if (!si || !cell || !parseAddress(address)) { + throw new Error(`Workbook contains a malformed shared formula group on ${patches[0]?.sheet}`); + } + const members = sharedGroups.get(si) ?? []; + members.push({ cell, formula, address }); + sharedGroups.set(si, members); + } + + const splices: XmlSplice[] = []; + for (const [si, members] of sharedGroups) { + if (!members.some((member) => patchAddresses.has(member.address))) continue; + const master = members.find((member) => !member.formula.selfClosing + && decodeXmlText(xml.slice(member.formula.openEnd, member.formula.closeStart)).length > 0); + if (!master) { + throw new Error(`Refusing to patch shared formula group ${si} on ${patches[0]?.sheet} because its master is missing`); + } + const masterFormula = decodeXmlText(xml.slice(master.formula.openEnd, master.formula.closeStart)); + if (!masterFormula) { + throw new Error(`Refusing to patch shared formula group ${si} on ${patches[0]?.sheet} because its master formula is empty`); + } + for (const member of members) { + if (patchAddresses.has(member.address)) continue; + const formula = member === master + ? masterFormula + : slideSharedFormula(masterFormula, master.address, member.address); + splices.push({ + start: member.formula.start, + end: member.formula.end, + text: `<${member.formula.name}>${escapeXmlText(formula)}`, + }); + } + } + return splices; +} + +function slideSharedFormula(formula: string, fromAddress: string, toAddress: string): string { + const from = parseAddress(fromAddress); + const to = parseAddress(toAddress); + if (!from || !to) throw new Error(`Cannot translate shared formula from ${fromAddress} to ${toAddress}`); + const candidate = /(([a-z_\-0-9]*)!)?([a-z0-9_$]{2,})([(])?/gi; + const address = /^([$])?([a-z]+)([$])?([1-9][0-9]*)$/i; + return formula.replace(candidate, (match, sheet: string | undefined, _sheetName: string | undefined, ref: string, call: string | undefined) => { + if (call) return match; + const parsed = address.exec(ref); + if (!parsed) return match; + const source = parseAddress(`${parsed[2]}${parsed[4]}`); + if (!source) return match; + const col = parsed[1] ? source.col : source.col + to.col - from.col; + const row = parsed[3] ? source.row : source.row + to.row - from.row; + if (col < 1 || row < 1 || col > 16_384 || row > 1_048_576) { + throw new Error(`Shared formula translation from ${fromAddress} to ${toAddress} produced an invalid reference`); + } + const translated = addressFromPosition(row, col); + const translatedMatch = /^([A-Z]+)([0-9]+)$/.exec(translated)!; + return `${sheet ?? ""}${parsed[1] ?? ""}${translatedMatch[1]}${parsed[3] ?? ""}${translatedMatch[2]}`; + }); +} + +function parseCellRange(value: string): { + ref: string; + minRow: number; + maxRow: number; + minCol: number; + maxCol: number; + area: number; +} | undefined { + const parts = value.trim().split(":"); + if (parts.length < 1 || parts.length > 2) return undefined; + const start = parseAddress(parts[0]); + const end = parseAddress(parts[1] ?? parts[0]); + if (!start || !end) return undefined; + const minRow = Math.min(start.row, end.row); + const maxRow = Math.max(start.row, end.row); + const minCol = Math.min(start.col, end.col); + const maxCol = Math.max(start.col, end.col); + return { + ref: `${addressFromPosition(minRow, minCol)}:${addressFromPosition(maxRow, maxCol)}`, + minRow, + maxRow, + minCol, + maxCol, + area: (maxRow - minRow + 1) * (maxCol - minCol + 1), + }; +} + +function coordinateInRange( + coordinate: CellCoordinate | undefined, + range: { minRow: number; maxRow: number; minCol: number; maxCol: number }, +): boolean { + return Boolean(coordinate + && coordinate.row >= range.minRow + && coordinate.row <= range.maxRow + && coordinate.col >= range.minCol + && coordinate.col <= range.maxCol); +} + +function serializeCellPatch( + xml: string, + patch: SpreadsheetBenchWorkbookCellPatch, + sourceCell: XmlNodeSpan | undefined, + sourceStyleIndex: number, + styles: WorkbookStylesEditor, +): string { + const formulaElement = patch.kind === "formula" + ? sourceFormulaElement(xml, sourceCell, patch) + : { opening: "", closing: "" }; + + const styleIndex = patch.numFmt === undefined + ? sourceStyleIndex + : styles.ensureNumberFormatStyle(sourceStyleIndex, patch.numFmt); + const tagName = sourceCell?.name ?? "c"; + let opening = sourceCell ? xml.slice(sourceCell.start, sourceCell.openEnd) : `<${tagName} r="${patch.address}">`; + opening = opening.replace(/\s*\/>$/, ">"); + opening = setXmlAttribute(opening, "r", patch.address); + opening = removeXmlAttribute(opening, "t"); + if (patch.numFmt !== undefined) { + opening = setXmlAttribute(opening, "s", String(styleIndex)); + } + + if (patch.kind === "clear") return opening.replace(/>$/, "/>"); + if (patch.kind === "formula") { + if (!patch.formula) throw new Error(`Formula patch ${patch.sheet}!${patch.address} is missing formula text`); + const cached = patch.hasCachedResult ? serializedScalar(patch.cachedResult, true) : undefined; + if (cached?.type) opening = setXmlAttribute(opening, "t", cached.type); + const cachedXml = cached ? `${cached.valueXml}` : ""; + return `${opening}${formulaElement.opening}${escapeXmlText(patch.formula.replace(/^=/, ""))}${formulaElement.closing}${cachedXml}`; + } + + const scalar = serializedScalar(patch.value, false); + if (scalar.type) opening = setXmlAttribute(opening, "t", scalar.type); + if (scalar.inlineString) { + const preserve = /^\s|\s$/.test(scalar.textValue ?? "") ? ' xml:space="preserve"' : ""; + return `${opening}${scalar.valueXml}`; + } + return `${opening}${scalar.valueXml}`; +} + +function sourceFormulaElement( + xml: string, + sourceCell: XmlNodeSpan | undefined, + patch: SpreadsheetBenchWorkbookCellPatch, +): { opening: string; closing: string } { + if (!sourceCell) return { opening: "", closing: "" }; + const sourceCellXml = xml.slice(sourceCell.start, sourceCell.end); + const formula = indexXmlSpans(sourceCellXml).find((node) => node.local === "f"); + if (!formula) return { opening: "", closing: "" }; + const formulaType = formula.attrs.t; + if (formulaType === "shared" || formulaType === "dataTable") { + return { opening: `<${formula.name}>`, closing: `` }; + } + if (formulaType === "array") { + const arrayRef = formula.attrs.ref?.split(":").map(normalizeAddress); + if (!arrayRef || arrayRef.length !== 1 || arrayRef[0] !== patch.address) { + return { opening: `<${formula.name}>`, closing: `` }; + } + return { + opening: sourceCellXml.slice(formula.start, formula.openEnd).replace(/\s*\/>$/, ">"), + closing: ``, + }; + } + return { opening: `<${formula.name}>`, closing: `` }; +} + +function serializedScalar(value: unknown, formulaResult: boolean): { + type?: string; + valueXml: string; + inlineString?: boolean; + textValue?: string; +} { + if (value === null || value === undefined) return { valueXml: "" }; + if (typeof value === "number") { + if (!Number.isFinite(value)) throw new Error(`Workbook patch cannot serialize non-finite number ${String(value)}`); + return { valueXml: String(value) }; + } + if (typeof value === "boolean") return { type: "b", valueXml: value ? "1" : "0" }; + if (value instanceof Date) { + const time = value.getTime(); + if (!Number.isFinite(time)) throw new Error("Workbook patch cannot serialize an invalid Date"); + return { valueXml: String((time - Date.UTC(1899, 11, 30)) / 86_400_000) }; + } + const record = asRecord(value); + if (typeof record?.error === "string") return { type: "e", valueXml: escapeXmlText(record.error) }; + const richText = Array.isArray(record?.richText) + ? record.richText.map((part) => { + const text = asRecord(part)?.text; + if (typeof text !== "string") throw new Error("Workbook patch rich-text runs must contain text"); + return text; + }).join("") + : undefined; + const text = typeof value === "string" + ? value + : typeof record?.text === "string" + ? record.text + : richText; + if (text === undefined) { + throw new Error(`Workbook patch cannot serialize ${Array.isArray(value) ? "an array" : typeof value} as a cell scalar`); + } + return formulaResult + ? { type: "str", valueXml: escapeXmlText(text), textValue: text } + : { type: "inlineStr", valueXml: escapeXmlText(text), inlineString: true, textValue: text }; +} + +function updateWorksheetDimension(xml: string): string { + const nodes = indexXmlSpans(xml); + const cells = nodes.filter((node) => node.local === "c" && parseAddress(node.attrs.r ?? "")); + if (cells.length === 0) return xml; + const coordinates = cells.map((cell) => parseAddress(cell.attrs.r ?? "")!); + const minRow = Math.min(...coordinates.map((coordinate) => coordinate.row)); + const maxRow = Math.max(...coordinates.map((coordinate) => coordinate.row)); + const minCol = Math.min(...coordinates.map((coordinate) => coordinate.col)); + const maxCol = Math.max(...coordinates.map((coordinate) => coordinate.col)); + const ref = minRow === maxRow && minCol === maxCol + ? addressFromPosition(minRow, minCol) + : `${addressFromPosition(minRow, minCol)}:${addressFromPosition(maxRow, maxCol)}`; + const dimension = nodes.find((node) => node.local === "dimension"); + if (!dimension) return xml; + const opening = setXmlAttribute(xml.slice(dimension.start, dimension.openEnd), "ref", ref); + return applyXmlSplices(xml, [{ start: dimension.start, end: dimension.openEnd, text: opening }]); +} + +async function markWorkbookForFullCalculation(zip: JSZip): Promise { + const workbookPath = "xl/workbook.xml"; + const workbook = await requiredZipText(zip, workbookPath); + + const nodes = indexXmlSpans(workbook); + const calcPr = nodes.find((node) => node.local === "calcPr"); + let nextWorkbook: string; + if (calcPr) { + let opening = workbook.slice(calcPr.start, calcPr.openEnd); + opening = setXmlAttribute(opening, "calcMode", "auto"); + opening = setXmlAttribute(opening, "fullCalcOnLoad", "1"); + opening = setXmlAttribute(opening, "forceFullCalc", "1"); + opening = setXmlAttribute(opening, "calcOnSave", "1"); + nextWorkbook = applyXmlSplices(workbook, [{ start: calcPr.start, end: calcPr.openEnd, text: opening }]); + } else { + const workbookNode = nodes.find((node) => node.local === "workbook"); + if (!workbookNode || workbookNode.selfClosing) throw new Error("Workbook XML has no writable workbook element"); + nextWorkbook = applyXmlSplices(workbook, [{ + start: workbookNode.closeStart, + end: workbookNode.closeStart, + text: '', + }]); + } + assertWellFormedXml(nextWorkbook, workbookPath); + zip.file(workbookPath, nextWorkbook); +} + +async function workbookSheetParts(zip: JSZip): Promise { + const workbookPath = "xl/workbook.xml"; + const workbookXml = await requiredZipText(zip, workbookPath); + const relsXml = await requiredZipText(zip, "xl/_rels/workbook.xml.rels"); + const relationships = new Map(); + for (const node of indexXmlSpans(relsXml).filter((candidate) => candidate.local === "Relationship")) { + if (!node.attrs.Id || !node.attrs.Target) continue; + if (node.attrs.TargetMode?.toLowerCase() === "external") continue; + if (node.attrs.Type && !node.attrs.Type.endsWith("/worksheet")) continue; + relationships.set(node.attrs.Id, resolvePackageTarget(workbookPath, node.attrs.Target)); + } + return indexXmlSpans(workbookXml) + .filter((node) => node.local === "sheet") + .flatMap((node) => { + const path = node.attrs["r:id"] ? relationships.get(node.attrs["r:id"]) : undefined; + return node.attrs.name && path ? [{ name: node.attrs.name, path }] : []; + }); +} + +function indexXmlSpans(xml: string): XmlNodeSpan[] { + const parser = new SaxesParser({ xmlns: true }); + const nodes: XmlNodeSpan[] = []; + const stack: XmlNodeSpan[] = []; + let pendingStart = 0; + parser.on("opentagstart", (tag) => { + pendingStart = parser.position - tag.name.length - 2; + }); + parser.on("opentag", (tag) => { + const node: XmlNodeSpan = { + name: tag.name, + local: tag.local, + attrs: saxesAttributes(tag), + start: pendingStart, + openEnd: parser.position, + closeStart: parser.position, + end: parser.position, + selfClosing: tag.isSelfClosing, + ...(stack.at(-1) ? { parent: stack.at(-1) } : {}), + }; + stack.push(node); + }); + parser.on("closetag", () => { + const node = stack.pop(); + if (!node) throw new Error("Malformed XML parser stack"); + node.end = parser.position; + node.closeStart = node.selfClosing ? node.openEnd : xml.lastIndexOf(" { + return Object.fromEntries(Object.values(tag.attributes).map((attribute) => [attribute.name, attribute.value])); +} + +function assertWellFormedXml(xml: string, label: string): void { + try { + indexXmlSpans(xml); + } catch (error) { + throw new Error(`${label} is not well-formed after workbook patching: ${error instanceof Error ? error.message : String(error)}`); + } +} + +function setElementAttribute(elementXml: string, name: string, value: string): string { + const end = elementXml.indexOf(">"); + if (end < 0) throw new Error("Malformed XML element"); + return `${setXmlAttribute(elementXml.slice(0, end + 1), name, value)}${elementXml.slice(end + 1)}`; +} + +function setXmlAttribute(openingTag: string, name: string, value: string): string { + const expression = new RegExp(`(\\s${escapeRegExp(name)}\\s*=\\s*)(["'])([\\s\\S]*?)\\2`, "i"); + if (expression.test(openingTag)) { + return openingTag.replace(expression, (_match, prefix: string) => `${prefix}"${escapeXmlAttribute(value)}"`); + } + return openingTag.replace(/\s*(\/?>)$/, ` ${name}="${escapeXmlAttribute(value)}"$1`); +} + +function removeXmlAttribute(openingTag: string, name: string): string { + return openingTag.replace(new RegExp(`\\s${escapeRegExp(name)}\\s*=\\s*(["'])[\\s\\S]*?\\1`, "i"), ""); +} + +function applyXmlSplices(xml: string, splices: XmlSplice[]): string { + const ordered = [...splices].sort((left, right) => right.start - left.start || right.end - left.end); + let previousStart = xml.length + 1; + let result = xml; + for (const splice of ordered) { + if (splice.start < 0 || splice.end < splice.start || splice.end > xml.length) throw new Error("Invalid XML splice bounds"); + if (splice.end > previousStart) throw new Error("Overlapping XML workbook patches are not allowed"); + result = `${result.slice(0, splice.start)}${splice.text}${result.slice(splice.end)}`; + previousStart = splice.start; + } + return result; +} + +function worksheetColumnStyles(nodes: XmlNodeSpan[]): ColumnStyleRange[] { + return nodes + .filter((node) => node.local === "col" && node.parent?.local === "cols") + .flatMap((node) => { + const min = numericAttribute(node.attrs.min); + const max = numericAttribute(node.attrs.max); + const style = numericAttribute(node.attrs.style); + return min && max && min <= max && style !== undefined ? [{ min, max, style }] : []; + }); +} + +function effectiveCellStyle( + cell: XmlNodeSpan | undefined, + row: XmlNodeSpan | undefined, + column: number, + columnStyles: ColumnStyleRange[], +): number { + const direct = numericAttribute(cell?.attrs.s); + if (direct !== undefined) return direct; + const rowStyle = numericAttribute(row?.attrs.s); + if (rowStyle !== undefined) return rowStyle; + for (let index = columnStyles.length - 1; index >= 0; index -= 1) { + const range = columnStyles[index]; + if (column >= range.min && column <= range.max) return range.style; + } + return 0; +} + +function numericAttribute(value: string | undefined): number | undefined { + if (value === undefined) return undefined; + const parsed = Number(value); + return Number.isInteger(parsed) && parsed >= 0 ? parsed : undefined; +} + +function parseAddress(value: string): CellCoordinate | undefined { + const match = normalizeAddress(value).match(/^([A-Z]{1,3})([1-9][0-9]*)$/); + if (!match) return undefined; + const row = Number(match[2]); + const col = match[1].split("").reduce((sum, char) => sum * 26 + char.charCodeAt(0) - 64, 0); + return row <= 1_048_576 && col <= 16_384 ? { row, col } : undefined; +} + +function addressFromPosition(row: number, col: number): string { + let name = ""; + let remaining = col; + while (remaining > 0) { + name = String.fromCharCode(65 + ((remaining - 1) % 26)) + name; + remaining = Math.floor((remaining - 1) / 26); + } + return `${name}${row}`; +} + +function normalizeAddress(value: string): string { + return value.trim().replace(/\$/g, "").toUpperCase(); +} + +async function requiredZipText(zip: JSZip, path: string): Promise { + const file = zip.file(path); + if (!file) throw new Error(`Workbook package is missing ${path}`); + return file.async("string"); +} + +function resolvePackageTarget(ownerPartPath: string, target: string): string { + const normalizedTarget = target.replace(/\\/g, "/"); + const joined = normalizedTarget.startsWith("/") + ? normalizedTarget + : posix.join(posix.dirname(ownerPartPath), normalizedTarget); + const normalized = posix.normalize(joined).replace(/^\/+/, ""); + if (!normalized || normalized === ".." || normalized.startsWith("../")) { + throw new Error(`Relationship target escapes the workbook package: ${target}`); + } + return normalized; +} + +function escapeXmlText(value: string): string { + return value.replace(/&/g, "&").replace(//g, ">"); +} + +function escapeXmlAttribute(value: string): string { + return escapeXmlText(value).replace(/"/g, """).replace(/'/g, "'"); +} + +function decodeXmlText(value: string): string { + return value.replace(/&(#x[0-9a-f]+|#[0-9]+|amp|lt|gt|quot|apos);/gi, (entity, token: string) => { + const lower = token.toLowerCase(); + if (lower === "amp") return "&"; + if (lower === "lt") return "<"; + if (lower === "gt") return ">"; + if (lower === "quot") return '"'; + if (lower === "apos") return "'"; + const codePoint = lower.startsWith("#x") + ? Number.parseInt(lower.slice(2), 16) + : Number.parseInt(lower.slice(1), 10); + return Number.isInteger(codePoint) ? String.fromCodePoint(codePoint) : entity; + }); +} + +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +function asRecord(value: unknown): Record | undefined { + return value && typeof value === "object" && !Array.isArray(value) ? value as Record : undefined; +} + +function atomicCopyFile(sourcePath: string, targetPath: string): void { + const temporaryPath = temporarySiblingPath(targetPath); + try { + copyFileSync(sourcePath, temporaryPath); + renameSync(temporaryPath, targetPath); + } catch (error) { + tryUnlink(temporaryPath); + throw error; + } +} + +function atomicWriteFile(targetPath: string, content: Buffer): void { + const temporaryPath = temporarySiblingPath(targetPath); + try { + writeFileSync(temporaryPath, content, { flag: "wx" }); + renameSync(temporaryPath, targetPath); + } catch (error) { + tryUnlink(temporaryPath); + throw error; + } +} + +function temporarySiblingPath(targetPath: string): string { + return join(dirname(targetPath), `.${basename(targetPath)}.${process.pid}.${randomUUID()}.tmp`); +} + +function tryUnlink(path: string): void { + try { + unlinkSync(path); + } catch { + // Best-effort cleanup after a failed atomic replace. + } +} diff --git a/tests/spreadsheetBenchNodeAgentBridge.test.ts b/tests/spreadsheetBenchNodeAgentBridge.test.ts index 7d84d4b6..1183fed7 100644 --- a/tests/spreadsheetBenchNodeAgentBridge.test.ts +++ b/tests/spreadsheetBenchNodeAgentBridge.test.ts @@ -131,7 +131,7 @@ describe("SpreadsheetBench canonical NodeAgent bridge", () => { }); }); - it("falls back to a sanitized workbook only when an unsupported WPS drawing blocks mutation", async () => { + it("emits the source workbook byte-for-byte when inspection makes no changes", async () => { const root = tempRoot(); const instruction = "Inspect the workbook and report what remains."; const task = await stagedTask(root, instruction); @@ -154,9 +154,73 @@ describe("SpreadsheetBench canonical NodeAgent bridge", () => { expect(receipt.stages.inspect.status).toBe("completed"); expect(receipt.frame.agentResult.stopReason).toBe("done"); expect(existsSync(candidate)).toBe(true); - const emitted = new ExcelJS.Workbook(); - await emitted.xlsx.readFile(candidate); - expect(emitted.getWorksheet("Model")?.getCell("A1").value).toBe(2); + expect(readFileSync(candidate)).toEqual(readFileSync(input)); + }); + + it("preserves unsupported workbook package parts while emitting changed cells", async () => { + const root = tempRoot(); + const task = await stagedTask(root); + const input = join(root, "tasks", "bridge-01", "agent", "inputs", "input.xlsx"); + await addUnsupportedWpsDrawing(input); + const candidate = join(root, "output", "wps-drawing-repaired.xlsx"); + const capture = { systems: [] as string[], messages: [] as string[], toolNames: [] as string[][] }; + + const receipt = await runSpreadsheetBenchNodeAgentBridge({ + agentManifestPath: task.agentManifest, + candidateWorkbookPath: candidate, + model: repairingWorkbookModel(capture), + traceId: "trace_sbench_wps_drawing_repaired", + maxSteps: 10, + now: () => 1_100, + }); + + expect(receipt.outcome).toMatchObject({ status: "completed", changedCellCount: 1 }); + const [sourceZip, candidateZip] = await Promise.all([ + JSZip.loadAsync(readFileSync(input)), + JSZip.loadAsync(readFileSync(candidate)), + ]); + for (const path of [ + "xl/drawings/drawing1.xml", + "xl/drawings/_rels/drawing1.xml.rels", + "xl/media/image1.png", + "xl/worksheets/_rels/sheet1.xml.rels", + ]) { + expect(await candidateZip.file(path)?.async("nodebuffer")).toEqual( + await sourceZip.file(path)?.async("nodebuffer"), + ); + } + const worksheetXml = await candidateZip.file("xl/worksheets/sheet1.xml")?.async("string"); + expect(worksheetXml).toContain(''); + expect(worksheetXml).toMatch(/]*r="B1"[^>]*>[\s\S]*?A1\*2<\/f>[\s\S]*?4<\/v>[\s\S]*?<\/c>/); + expect(await candidateZip.file("[Content_Types].xml")?.async("string")).toContain( + 'PartName="/xl/drawings/drawing1.xml"', + ); + }); + + it("treats invalid Excel dates as inspectable values instead of aborting the trace", async () => { + const root = tempRoot(); + const task = await stagedTask(root, "Inspect the workbook and report what remains."); + const inputPath = join(root, "tasks", "bridge-01", "agent", "inputs", "input.xlsx"); + const input = new ExcelJS.Workbook(); + await input.xlsx.readFile(inputPath); + input.getWorksheet("Model")!.getCell("A1").value = 1e100; + input.getWorksheet("Model")!.getCell("A1").numFmt = "yyyy-mm-dd"; + await input.xlsx.writeFile(inputPath); + const candidate = join(root, "output", "invalid-date.xlsx"); + + const receipt = await runSpreadsheetBenchNodeAgentBridge({ + agentManifestPath: task.agentManifest, + candidateWorkbookPath: candidate, + model: invalidArtifactInspectionModel(), + traceId: "trace_sbench_invalid_date", + maxSteps: 4, + now: () => 1_150, + }); + + expect(receipt.frame.runtimeError).toBeUndefined(); + expect(receipt.stages.inspect.status).toBe("completed"); + expect(receipt.outcome.changedCellCount).toBe(0); + expect(readFileSync(candidate)).toEqual(readFileSync(inputPath)); }); it("keeps a syntactically verified write open when deterministic recalculation is unsupported", async () => { diff --git a/tests/spreadsheetBenchScorer.test.ts b/tests/spreadsheetBenchScorer.test.ts index f6584239..526ff559 100644 --- a/tests/spreadsheetBenchScorer.test.ts +++ b/tests/spreadsheetBenchScorer.test.ts @@ -1,4 +1,5 @@ import ExcelJS from "exceljs"; +import JSZip from "jszip"; import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { tmpdir } from "node:os"; @@ -130,6 +131,34 @@ describe("SpreadsheetBench workbook scorer", () => { expect(score.scores).toMatchObject({ value: 1, formula: null, overall: 1 }); }); + it("parses self-closing and explicitly closed blank cells without consuming the following cell", async () => { + const root = tempRoot(); + const candidate = join(root, "candidate.xlsx"); + const gold = join(root, "gold.xlsx"); + await writeBlankThenFormulaWorkbook(candidate); + await writeBlankThenFormulaWorkbook(gold); + await expandSelfClosingCell(candidate, "A1"); + await encodeSharedStringWithNumericReferences(candidate, "Line one\nLine two ©"); + + const score = await scoreSpreadsheetBenchWorkbook({ + taskId: "fixture/self-closing-cell", + candidateWorkbookPath: candidate, + goldWorkbookPath: gold, + answerPosition: "'Model'!A1:D1", + maxMismatches: 10, + generatedAt: "2026-06-13T00:00:00.000Z", + }); + + expect(score.pass).toBe(true); + expect(score.totals).toMatchObject({ + comparedCells: 4, + valueMatches: 4, + formulaCells: 1, + formulaMatches: 1, + mismatches: 0, + }); + }); + it("scores row, column, and merge layout drift when style comparison is enabled", async () => { const root = tempRoot(); const candidate = join(root, "candidate.xlsx"); @@ -251,6 +280,40 @@ async function writeFormulaVsScalarWorkbook(path: string, mode: "formula" | "sca await workbook.xlsx.writeFile(path); } +async function writeBlankThenFormulaWorkbook(path: string) { + const workbook = new ExcelJS.Workbook(); + const sheet = workbook.addWorksheet("Model"); + sheet.getCell("A1").numFmt = "0"; + sheet.getCell("B1").value = 2; + sheet.getCell("C1").value = { formula: "B1*2", result: 4 }; + sheet.getCell("D1").value = "Line one\nLine two ©"; + await workbook.xlsx.writeFile(path); +} + +async function expandSelfClosingCell(path: string, address: string) { + const zip = await JSZip.loadAsync(readFileSync(path)); + const worksheet = zip.file("xl/worksheets/sheet1.xml"); + if (!worksheet) throw new Error("fixture worksheet is missing"); + const xml = await worksheet.async("string"); + const expression = new RegExp(`]*\\br=["']${address}["'][^>]*)\\/>`, "i"); + const expanded = xml.replace(expression, (_match, attrs: string) => ``); + if (expanded === xml) throw new Error(`fixture cell ${address} was not self-closing`); + zip.file("xl/worksheets/sheet1.xml", expanded); + writeFileSync(path, await zip.generateAsync({ type: "nodebuffer" })); +} + +async function encodeSharedStringWithNumericReferences(path: string, value: string) { + const zip = await JSZip.loadAsync(readFileSync(path)); + const sharedStrings = zip.file("xl/sharedStrings.xml"); + if (!sharedStrings) throw new Error("fixture shared strings are missing"); + const xml = await sharedStrings.async("string"); + const encoded = value.replace("\n", " ").replace("©", "©"); + const patched = xml.replace(value, encoded); + if (patched === xml) throw new Error("fixture shared string was not found"); + zip.file("xl/sharedStrings.xml", patched); + writeFileSync(path, await zip.generateAsync({ type: "nodebuffer" })); +} + async function writeLayoutWorkbook(path: string, args: { formatted: boolean }) { const workbook = new ExcelJS.Workbook(); const sheet = workbook.addWorksheet("Model"); diff --git a/tests/spreadsheetBenchWorkbookEmitter.test.ts b/tests/spreadsheetBenchWorkbookEmitter.test.ts new file mode 100644 index 00000000..0193c9f2 --- /dev/null +++ b/tests/spreadsheetBenchWorkbookEmitter.test.ts @@ -0,0 +1,365 @@ +import ExcelJS from "exceljs"; +import JSZip from "jszip"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { emitSpreadsheetBenchWorkbookCandidate } from "../src/eval/spreadsheetBenchWorkbookEmitter"; + +const roots: string[] = []; +const WORKSHEET_PATH = "xl/customSheets/model-data.xml"; +const FIXED_ZIP_DATE = new Date("2020-01-02T03:04:06.000Z"); + +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); + +describe("SpreadsheetBench workbook emitter", () => { + it("copies the source workbook byte-for-byte when there are no patches", async () => { + const root = tempRoot(); + const source = join(root, "source.xlsx"); + const candidate = join(root, "candidate.xlsx"); + await writeOoxmlFixture(source); + + await emitSpreadsheetBenchWorkbookCandidate({ + sourceWorkbookPath: source, + candidateWorkbookPath: candidate, + patches: [], + }); + + expect(readFileSync(candidate)).toEqual(readFileSync(source)); + }); + + it("resolves the worksheet relationship and preserves unrelated package parts byte-for-byte", async () => { + const root = tempRoot(); + const source = join(root, "source.xlsx"); + const candidate = join(root, "candidate.xlsx"); + await writeOoxmlFixture(source); + + await emitSpreadsheetBenchWorkbookCandidate({ + sourceWorkbookPath: source, + candidateWorkbookPath: candidate, + patches: [{ sheet: "model", address: "$b$2", kind: "value", value: 99 }], + }); + + const [sourceZip, candidateZip] = await Promise.all([ + JSZip.loadAsync(readFileSync(source)), + JSZip.loadAsync(readFileSync(candidate)), + ]); + expect(packagePartPaths(candidateZip)).toEqual(packagePartPaths(sourceZip)); + + for (const path of [ + "_rels/.rels", + "docProps/core.xml", + "customXml/item1.xml", + "xl/_rels/workbook.xml.rels", + "xl/calcChain.xml", + "xl/customSheets/_rels/model-data.xml.rels", + "xl/drawings/drawing7.xml", + "xl/drawings/_rels/drawing7.xml.rels", + "xl/charts/chart7.xml", + "xl/media/image7.png", + ]) { + expect(await requiredZipBytes(candidateZip, path)).toEqual(await requiredZipBytes(sourceZip, path)); + } + + const sourceWorksheet = await requiredZipText(sourceZip, WORKSHEET_PATH); + const candidateWorksheet = await requiredZipText(candidateZip, WORKSHEET_PATH); + expect(candidateWorksheet).not.toBe(sourceWorksheet); + expect(candidateWorksheet).toContain('99'); + }); + + it("persists patched formulas, cached values, scalar values, and source styles", async () => { + const root = tempRoot(); + const source = join(root, "styled-source.xlsx"); + const candidate = join(root, "styled-candidate.xlsx"); + await writeStyledWorkbook(source); + + await emitSpreadsheetBenchWorkbookCandidate({ + sourceWorkbookPath: source, + candidateWorkbookPath: candidate, + patches: [ + { + sheet: "Model", + address: "B2", + kind: "formula", + formula: "=A1*3", + hasCachedResult: true, + cachedResult: 30, + numFmt: "0.000%", + }, + { sheet: "Model", address: "C3", kind: "value", value: " updated " }, + ], + }); + + const emitted = new ExcelJS.Workbook(); + await emitted.xlsx.readFile(candidate); + const sheet = emitted.getWorksheet("Model"); + expect(sheet).toBeDefined(); + + const formulaCell = sheet!.getCell("B2"); + expect(formulaCell.value).toEqual({ formula: "A1*3", result: 30 }); + expect(formulaCell.numFmt).toBe("0.000%"); + expect(formulaCell.font).toMatchObject({ bold: true, color: { argb: "FF123456" } }); + expect(formulaCell.fill).toMatchObject({ + type: "pattern", + pattern: "solid", + fgColor: { argb: "FFFFFF00" }, + }); + expect(formulaCell.border.bottom).toMatchObject({ style: "thin", color: { argb: "FF654321" } }); + + const valueCell = sheet!.getCell("C3"); + expect(valueCell.value).toBe(" updated "); + expect(valueCell.numFmt).toBe("@"); + expect(valueCell.font).toMatchObject({ italic: true }); + expect(valueCell.alignment).toMatchObject({ horizontal: "center" }); + + const untouchedCell = sheet!.getCell("D4"); + expect(untouchedCell.value).toEqual({ formula: "A1+5", result: 15 }); + expect(untouchedCell.numFmt).toBe("0.00"); + expect(untouchedCell.font).toMatchObject({ underline: true }); + }); + + it("materializes untouched shared-formula members before rewriting one member", async () => { + const root = tempRoot(); + const source = join(root, "shared-source.xlsx"); + const candidate = join(root, "shared-candidate.xlsx"); + await writeOoxmlFixture(source, { + formulaXml: 'A1*2', + extraRowsXml: '4', + }); + + await emitSpreadsheetBenchWorkbookCandidate({ + sourceWorkbookPath: source, + candidateWorkbookPath: candidate, + patches: [{ sheet: "Model", address: "B2", kind: "formula", formula: "=A1*4" }], + }); + + const zip = await JSZip.loadAsync(readFileSync(candidate)); + const worksheet = await requiredZipText(zip, WORKSHEET_PATH); + expect(worksheet).toContain('A1*4'); + expect(worksheet).toContain('A2*24'); + expect(worksheet).not.toContain('t="shared"'); + }); + + it.each([ + { + label: "multi-cell array formula", + formulaXml: 'A1*2', + row2TailXml: '4', + extraRowsXml: "", + expectedError: "Refusing to patch array formula range Model!B2:C2 without all 2 cells", + }, + { + label: "data-table formula group", + formulaXml: '', + row2TailXml: '4', + extraRowsXml: '44', + expectedError: "Refusing to patch dataTable formula range Model!B2:C3 without all 4 cells", + }, + ])("refuses to rewrite a cell inside a $label", async ({ formulaXml, row2TailXml, extraRowsXml, expectedError }) => { + const root = tempRoot(); + const source = join(root, "grouped-source.xlsx"); + const candidate = join(root, "grouped-candidate.xlsx"); + await writeOoxmlFixture(source, { formulaXml, row2TailXml, extraRowsXml }); + + await expect(emitSpreadsheetBenchWorkbookCandidate({ + sourceWorkbookPath: source, + candidateWorkbookPath: candidate, + patches: [{ sheet: "Model", address: "B2", kind: "formula", formula: "=A1*4" }], + })).rejects.toThrow(expectedError); + }); + + it("allows a single-cell array formula rewrite while retaining its array metadata", async () => { + const root = tempRoot(); + const source = join(root, "array-source.xlsx"); + const candidate = join(root, "array-candidate.xlsx"); + await writeOoxmlFixture(source, { formulaXml: 'A1*2' }); + + await emitSpreadsheetBenchWorkbookCandidate({ + sourceWorkbookPath: source, + candidateWorkbookPath: candidate, + patches: [{ + sheet: "Model", + address: "B2", + kind: "formula", + formula: "=A1*4", + hasCachedResult: true, + cachedResult: 8, + }], + }); + + const zip = await JSZip.loadAsync(readFileSync(candidate)); + expect(await requiredZipText(zip, WORKSHEET_PATH)).toContain( + 'A1*48', + ); + }); +}); + +function tempRoot(): string { + const root = mkdtempSync(join(tmpdir(), "noderoom-spreadsheetbench-emitter-")); + roots.push(root); + return root; +} + +async function writeStyledWorkbook(path: string): Promise { + const workbook = new ExcelJS.Workbook(); + workbook.creator = "NodeRoom tests"; + workbook.created = new Date("2020-01-02T03:04:06.000Z"); + workbook.modified = new Date("2020-01-02T03:04:06.000Z"); + const sheet = workbook.addWorksheet("Model"); + sheet.getCell("A1").value = 10; + + const formulaCell = sheet.getCell("B2"); + formulaCell.value = { formula: "A1*2", result: 20 }; + formulaCell.numFmt = "$#,##0.00"; + formulaCell.font = { bold: true, color: { argb: "FF123456" } }; + formulaCell.fill = { type: "pattern", pattern: "solid", fgColor: { argb: "FFFFFF00" } }; + formulaCell.border = { bottom: { style: "thin", color: { argb: "FF654321" } } }; + + const valueCell = sheet.getCell("C3"); + valueCell.value = "before"; + valueCell.numFmt = "@"; + valueCell.font = { italic: true }; + valueCell.alignment = { horizontal: "center" }; + + const untouchedCell = sheet.getCell("D4"); + untouchedCell.value = { formula: "A1+5", result: 15 }; + untouchedCell.numFmt = "0.00"; + untouchedCell.font = { underline: true }; + await workbook.xlsx.writeFile(path); +} + +async function writeOoxmlFixture(path: string, options: { + formulaXml?: string; + row2TailXml?: string; + extraRowsXml?: string; +} = {}): Promise { + const zip = new JSZip(); + const add = (partPath: string, data: string | Buffer) => { + zip.file(partPath, data, { createFolders: false, date: FIXED_ZIP_DATE }); + }; + const formulaXml = options.formulaXml ?? "A1*2"; + + add( + "[Content_Types].xml", + '' + + '' + + '' + + '' + + '' + + '' + + `` + + '' + + '' + + '' + + '' + + "", + ); + add( + "_rels/.rels", + '' + + '' + + '' + + "", + ); + add( + "docProps/core.xml", + '7', + ); + add("customXml/item1.xml", 'relationship/chart/drawing/media'); + add( + "xl/workbook.xml", + '' + + '' + + '' + + '' + + "", + ); + add( + "xl/_rels/workbook.xml.rels", + '' + + '' + + '' + + '' + + '' + + "", + ); + add( + "xl/styles.xml", + '' + + '' + + '' + + '' + + '' + + '' + + '' + + '' + + "", + ); + add( + WORKSHEET_PATH, + '' + + '' + + '' + + '2' + + `${formulaXml}4${options.row2TailXml ?? ""}` + + `${options.extraRowsXml ?? ""}`, + ); + add( + "xl/calcChain.xml", + '', + ); + add( + "xl/customSheets/_rels/model-data.xml.rels", + '' + + '' + + '' + + "", + ); + add( + "xl/drawings/drawing7.xml", + '' + + '' + + '' + + "", + ); + add( + "xl/drawings/_rels/drawing7.xml.rels", + '' + + '' + + '' + + '' + + "", + ); + add( + "xl/charts/chart7.xml", + '', + ); + add("xl/media/image7.png", Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0xff, 0x7f])); + + writeFileSync(path, await zip.generateAsync({ + type: "nodebuffer", + compression: "DEFLATE", + compressionOptions: { level: 6 }, + })); +} + +async function requiredZipText(zip: JSZip, path: string): Promise { + const file = zip.file(path); + if (!file) throw new Error(`Fixture package is missing ${path}`); + return file.async("string"); +} + +async function requiredZipBytes(zip: JSZip, path: string): Promise { + const file = zip.file(path); + if (!file) throw new Error(`Fixture package is missing ${path}`); + return file.async("nodebuffer"); +} + +function packagePartPaths(zip: JSZip): string[] { + return Object.values(zip.files) + .filter((entry) => !entry.dir) + .map((entry) => entry.name) + .sort(); +} diff --git a/tests/spreadsheetbench_refresh_excel_test.py b/tests/spreadsheetbench_refresh_excel_test.py new file mode 100644 index 00000000..3f689edd --- /dev/null +++ b/tests/spreadsheetbench_refresh_excel_test.py @@ -0,0 +1,292 @@ +from __future__ import annotations + +import importlib.util +import sys +import tempfile +import unittest +import zipfile +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[1] +SCRIPT_PATH = REPO_ROOT / "scripts" / "spreadsheetbench-refresh-excel.py" +MODULE_NAME = "spreadsheetbench_refresh_excel" +SPEC = importlib.util.spec_from_file_location(MODULE_NAME, SCRIPT_PATH) +if SPEC is None or SPEC.loader is None: + raise RuntimeError(f"could not load {SCRIPT_PATH}") +refresh_excel = importlib.util.module_from_spec(SPEC) +sys.modules[MODULE_NAME] = refresh_excel +SPEC.loader.exec_module(refresh_excel) + + +WORKSHEET_NS = "http://schemas.openxmlformats.org/spreadsheetml/2006/main" +DOCUMENT_REL_NS = "http://schemas.openxmlformats.org/officeDocument/2006/relationships" +PACKAGE_REL_NS = "http://schemas.openxmlformats.org/package/2006/relationships" +WORKSHEET_REL_TYPE = f"{DOCUMENT_REL_NS}/worksheet" + + +def write_zip( + path: Path, + entries: list[tuple[str, str | bytes, bytes]], + *, + comment: bytes = b"", +) -> None: + with zipfile.ZipFile(path, "w") as package: + package.comment = comment + for name, data, entry_comment in entries: + info = zipfile.ZipInfo(name, date_time=(2020, 1, 2, 3, 4, 6)) + info.compress_type = zipfile.ZIP_DEFLATED + info.comment = entry_comment + info.external_attr = 0o600 << 16 + package.writestr(info, data) + + +class FormulaCacheSerializationTests(unittest.TestCase): + def assert_cache(self, value: object, cell_type: str | None, value_xml: str) -> None: + self.assertEqual( + refresh_excel.serialize_formula_cache(value), + refresh_excel.FormulaCache(cell_type, value_xml), + ) + + def test_serializes_numbers_booleans_and_empty_values(self) -> None: + self.assert_cache(None, None, "") + self.assert_cache(True, "b", "1") + self.assert_cache(False, "b", "0") + self.assert_cache(0, None, "0") + self.assert_cache(-42, None, "-42") + self.assert_cache(1.25, None, "1.25") + + def test_serializes_classic_and_modern_excel_errors(self) -> None: + for code, text in { + 2000: "#NULL!", + 2007: "#DIV/0!", + 2023: "#REF!", + 2042: "#N/A", + 2045: "#SPILL!", + 2049: "#DATA!", + 2053: "#PYTHON!", + }.items(): + with self.subTest(code=code): + self.assert_cache(code, "e", text) + self.assert_cache(0x800A0000 | code, "e", text) + + def test_encodes_control_characters_cr_and_literal_excel_escapes(self) -> None: + value = "<&\x00\x01\t\n\r\x0b> _x0041_ _X000D_" + self.assert_cache( + value, + "str", + "<&_x0000__x0001_\t\n_x000D__x000B_> " + "_x005F_x0041_ _x005F_X000D_", + ) + + def test_rejects_non_finite_numeric_values(self) -> None: + for value in (float("nan"), float("inf"), float("-inf")): + with self.subTest(value=value): + with self.assertRaisesRegex(RuntimeError, "non-finite"): + refresh_excel.serialize_formula_cache(value) + + +class FormulaTargetDiscoveryTests(unittest.TestCase): + def test_discovers_formula_shapes_and_resolves_sheet_relationships(self) -> None: + workbook_xml = ( + f'' + "" + '' + '' + "" + "" + ) + relationships_xml = ( + f'' + f'' + f'' + "" + ) + relative_sheet_xml = ( + f'' + '1+12' + 'A3*2' + "2" + '4' + '6' + 'SEQUENCE(2,2)' + "1" + '' + "" + ) + absolute_sheet_xml = ( + f'' + '10' + '' + "" + ) + + with tempfile.TemporaryDirectory() as temp_dir: + workbook_path = Path(temp_dir) / "formula-targets.xlsx" + write_zip( + workbook_path, + [ + ("xl/workbook.xml", workbook_xml, b""), + ("xl/_rels/workbook.xml.rels", relationships_xml, b""), + ("xl/worksheets/relative.xml", relative_sheet_xml, b""), + ("xl/custom/absolute.xml", absolute_sheet_xml, b""), + ], + ) + + targets = refresh_excel.workbook_formula_cells(workbook_path) + + self.assertEqual( + targets, + { + "Relative formulas": ( + "xl/worksheets/relative.xml", + ["B2", "C3", "C4", "D6", "E6", "D7", "E7"], + ), + "Absolute formulas": ( + "xl/custom/absolute.xml", + ["G10", "H10", "G11", "H11"], + ), + }, + ) + self.assertNotIn("C5", targets["Relative formulas"][1]) + + def test_sparse_formula_areas_keep_tight_excel_range_bounds(self) -> None: + addresses = [ + "XFD1048576", + "B2", + "A1", + "F4", + "B1", + "A2", + "D4", + "A1", + ] + + self.assertEqual( + refresh_excel.contiguous_formula_areas(addresses), + [ + (1, 2, 1, 2), + (4, 4, 4, 4), + (4, 4, 6, 6), + (1048576, 1048576, 16384, 16384), + ], + ) + self.assertEqual(refresh_excel.contiguous_formula_areas([]), []) + + +class WorksheetCachePatchingTests(unittest.TestCase): + def test_patches_formula_shapes_without_rewriting_unrelated_xml(self) -> None: + source_xml = ( + '' + f'' + '' + 'SUM( A3:A4 )&<literal>' + 'stale-anchor' + '' + '' + 'UNTOUCHED()99' + "" + "" + ) + expected_xml = ( + '' + f'' + '' + 'SUM( A3:A4 )&<literal>' + '12.5' + '0' + 'ready & waiting' + 'UNTOUCHED()99' + "" + "" + ) + + patched_xml, patched_addresses = refresh_excel.patch_worksheet_formula_caches( + source_xml, + {"A1": 12.5, "A2": False, "B2": "ready & waiting"}, + ) + + self.assertEqual(patched_addresses, {"A1", "A2", "B2"}) + self.assertEqual(patched_xml, expected_xml) + + +class WorkbookPackagePatchingTests(unittest.TestCase): + def make_workbook(self, path: Path) -> str: + worksheet_xml = ( + f'' + '40+2stale' + 'unrelated' + "" + ) + write_zip( + path, + [ + ("[Content_Types].xml", "", b"types-entry-comment"), + ("xl/workbook.xml", "", b""), + ("xl/worksheets/sheet1.xml", worksheet_xml, b"sheet-entry-comment"), + ("docProps/custom.bin", b"\x00\x01unchanged\xff", b"binary-entry-comment"), + ], + comment=b"workbook archive comment", + ) + return worksheet_xml + + def test_preserves_zip_entries_comments_and_unrelated_content(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + workbook_path = Path(temp_dir) / "book.xlsx" + original_worksheet = self.make_workbook(workbook_path) + with zipfile.ZipFile(workbook_path, "r") as package: + original_names = package.namelist() + original_comment = package.comment + original_binary = package.read("docProps/custom.bin") + original_entry_comments = { + info.filename: info.comment for info in package.infolist() + } + + refresh_excel.patch_formula_caches( + workbook_path, + {"Sheet 1": ("xl/worksheets/sheet1.xml", ["A1"])}, + {"Sheet 1": {"A1": 42}}, + expected_sha256=refresh_excel.sha256_file(workbook_path), + ) + + with zipfile.ZipFile(workbook_path, "r") as package: + patched_worksheet = package.read("xl/worksheets/sheet1.xml").decode("utf-8") + self.assertEqual(package.namelist(), original_names) + self.assertEqual(package.comment, original_comment) + self.assertEqual(package.read("docProps/custom.bin"), original_binary) + self.assertEqual( + {info.filename: info.comment for info in package.infolist()}, + original_entry_comments, + ) + self.assertIsNone(package.testzip()) + + self.assertNotEqual(patched_worksheet, original_worksheet) + self.assertIn("40+242", patched_worksheet) + self.assertIn('unrelated', patched_worksheet) + + def test_rejects_stale_expected_sha_without_modifying_the_workbook(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + workbook_path = Path(temp_dir) / "book.xlsx" + self.make_workbook(workbook_path) + stale_sha = refresh_excel.sha256_file(workbook_path) + with workbook_path.open("ab") as stream: + stream.write(b"changed-after-hash") + bytes_before_patch_attempt = workbook_path.read_bytes() + + with self.assertRaisesRegex(RuntimeError, "changed before cache patching"): + refresh_excel.patch_formula_caches( + workbook_path, + {"Sheet 1": ("xl/worksheets/sheet1.xml", ["A1"])}, + {"Sheet 1": {"A1": 42}}, + expected_sha256=stale_sha, + ) + + self.assertEqual(workbook_path.read_bytes(), bytes_before_patch_attempt) + self.assertEqual([path.name for path in Path(temp_dir).iterdir()], ["book.xlsx"]) + + +if __name__ == "__main__": + unittest.main() From a40746b7b3bc2f79782adabe08d6e220ef446c23 Mon Sep 17 00:00:00 2001 From: homen Date: Wed, 15 Jul 2026 06:33:45 -0700 Subject: [PATCH 10/12] fix(scripts src tests): seal SpreadsheetBench evidence after cache refresh --- .../spreadsheetbench-merge-repair-report.ts | 1 + scripts/spreadsheetbench-refresh-excel.py | 24 ++++- scripts/spreadsheetbench-run-chunked.ts | 7 +- scripts/spreadsheetbench-run.ts | 7 +- src/eval/spreadsheetBenchNodeAgentBridge.ts | 62 +++++++++++++ src/eval/spreadsheetBenchRunner.ts | 88 ++++++++++++++++++- tests/spreadsheetBenchNodeAgentBridge.test.ts | 62 +++++++++++++ tests/spreadsheetbench_refresh_excel_test.py | 34 +++++++ 8 files changed, 279 insertions(+), 6 deletions(-) diff --git a/scripts/spreadsheetbench-merge-repair-report.ts b/scripts/spreadsheetbench-merge-repair-report.ts index 96d7edd6..f38799f3 100644 --- a/scripts/spreadsheetbench-merge-repair-report.ts +++ b/scripts/spreadsheetbench-merge-repair-report.ts @@ -168,6 +168,7 @@ function resultEvidence(result: SpreadsheetBenchRunnerTaskResult): SpreadsheetBe "rawModelOutput", "workbookInspection", "editVerification", + "candidateFinalization", ] as const; for (const key of optionalFileKeys) { const value = sidecars[key]; diff --git a/scripts/spreadsheetbench-refresh-excel.py b/scripts/spreadsheetbench-refresh-excel.py index 562c355b..221f511d 100644 --- a/scripts/spreadsheetbench-refresh-excel.py +++ b/scripts/spreadsheetbench-refresh-excel.py @@ -611,18 +611,36 @@ def create_excel_application(): raise +def select_workbooks(directory: str | None, file: str | None) -> tuple[Path, list[Path]]: + if bool(directory) == bool(file): + raise ValueError("exactly one of --dir or --file is required") + if file: + workbook = Path(file).resolve() + if not workbook.is_file() or workbook.suffix.lower() != ".xlsx": + raise ValueError(f"--file must name an existing .xlsx workbook: {workbook}") + return workbook.parent, [workbook] + root = Path(str(directory)).resolve() + if not root.is_dir(): + raise ValueError(f"--dir must name an existing directory: {root}") + return root, sorted(root.rglob("*_output.xlsx")) + + def main() -> int: parser = argparse.ArgumentParser() - parser.add_argument("--dir", required=True, help="Root containing *_output.xlsx files") + source = parser.add_mutually_exclusive_group(required=True) + source.add_argument("--dir", help="Root containing *_output.xlsx files") + source.add_argument("--file", help="One candidate .xlsx workbook to refresh before evidence sealing") parser.add_argument("--receipt", required=True, help="JSON receipt path") parser.add_argument("--retry-receipt", help="Retry only failed records from this receipt and merge the results") args = parser.parse_args() - root = Path(args.dir).resolve() + try: + root, files = select_workbooks(args.dir, args.file) + except ValueError as exc: + parser.error(str(exc)) receipt_path = Path(args.receipt).resolve() previous_receipt: dict[str, object] | None = None previous_records: dict[str, dict[str, object]] = {} - files = sorted(root.rglob("*_output.xlsx")) if args.retry_receipt: previous_receipt = json.loads(Path(args.retry_receipt).resolve().read_text(encoding="utf-8")) previous_records = {str(record["path"]): record for record in previous_receipt.get("records", [])} diff --git a/scripts/spreadsheetbench-run-chunked.ts b/scripts/spreadsheetbench-run-chunked.ts index 69f88be8..e1c070af 100644 --- a/scripts/spreadsheetbench-run-chunked.ts +++ b/scripts/spreadsheetbench-run-chunked.ts @@ -38,6 +38,7 @@ const taskIds = taskIdsFile ? readTaskIds(taskIdsFile) : undefined; const compareStyles = args.includes("--compare-styles"); const compareCharts = args.includes("--compare-charts"); const retryScoreFailures = args.includes("--retry-score-failures"); +const refreshExcelCaches = args.includes("--refresh-excel-caches"); const clean = args.includes("--clean"); const resume = args.includes("--resume"); const repairMissingModelReceipts = args.includes("--repair-missing-model-receipts"); @@ -56,7 +57,7 @@ const modeRequiresModel = mode === "model-edit-plan" || mode === "nodeagent-work if (!stageRoot || !outputRoot || !jsonOut || !allowedModes.includes(mode) || chunkSize <= 0 || concurrency <= 0 || concurrency > 16) { console.error([ "Usage:", - " npm run benchmark:spreadsheetbench:run-chunked -- --stage-root --output-root --json-out [--mode copy-input-baseline|apply-agent-patch|model-edit-plan|nodeagent-workbook] [--chunk-size 25] [--concurrency 1..16] [--resume] [--repair-missing-model-receipts] [--model ] [--free-auto-mode chat|agent|structured|vision|coding] [--model-batch-size 1] [--model-snapshot-max-cells 800] [--model-snapshot-max-cell-chars 256] [--model-repair-attempts 1] [--task-ids-file ] [--allow-provider-spend --max-provider-cost-usd 1 --provider-call-reserve-usd 0.05] [--clean]", + " npm run benchmark:spreadsheetbench:run-chunked -- --stage-root --output-root --json-out [--mode copy-input-baseline|apply-agent-patch|model-edit-plan|nodeagent-workbook] [--chunk-size 25] [--concurrency 1..16] [--resume] [--repair-missing-model-receipts] [--model ] [--free-auto-mode chat|agent|structured|vision|coding] [--model-batch-size 1] [--model-snapshot-max-cells 800] [--model-snapshot-max-cell-chars 256] [--model-repair-attempts 1] [--refresh-excel-caches] [--task-ids-file ] [--allow-provider-spend --max-provider-cost-usd 1 --provider-call-reserve-usd 0.05] [--clean]", "", "Runs staged SpreadsheetBench tasks in fresh child processes and aggregates the reports.", "nodeagent-workbook requires --model, defaults --free-auto-mode to agent, and requires --model-batch-size 1.", @@ -84,6 +85,9 @@ if (mode === "nodeagent-workbook" && modelBatchSize > 1) throw new Error("nodeag if (modelSnapshotMaxCells !== undefined && modelSnapshotMaxCells < 1) throw new Error("--model-snapshot-max-cells must be at least 1."); if (modelSnapshotMaxCellChars !== undefined && modelSnapshotMaxCellChars < 1) throw new Error("--model-snapshot-max-cell-chars must be at least 1."); if (modelRepairAttempts < 0 || modelRepairAttempts > 3) throw new Error("--model-repair-attempts must be between 0 and 3."); +if (refreshExcelCaches && mode !== "nodeagent-workbook") { + throw new Error("--refresh-excel-caches requires --mode nodeagent-workbook."); +} if (providerCallReserveUsd <= 0) throw new Error("--provider-call-reserve-usd must be positive."); if (maxProviderCostUsd !== undefined && maxProviderCostUsd <= 0) throw new Error("--max-provider-cost-usd must be positive."); if (allowProviderSpend && maxProviderCostUsd === undefined) throw new Error("--allow-provider-spend requires --max-provider-cost-usd."); @@ -181,6 +185,7 @@ async function runChunk(index: number, offset: number, limit: number): Promise --output-root [--mode copy-input-baseline|apply-agent-patch|model-edit-plan|nodeagent-workbook] [--model ] [--free-auto-mode chat|agent|structured|vision|coding] [--model-batch-size 1] [--model-snapshot-max-cells 800] [--model-snapshot-max-cell-chars 256] [--model-repair-attempts 1] [--task-ids-file ] [--offset 0] [--limit 3] [--repeats 5] [--retry-failed 2] [--retry-score-failures] [--compare-charts] [--clean] [--json-out ]", + " npm run benchmark:spreadsheetbench:run -- --stage-root --output-root [--mode copy-input-baseline|apply-agent-patch|model-edit-plan|nodeagent-workbook] [--model ] [--free-auto-mode chat|agent|structured|vision|coding] [--model-batch-size 1] [--model-snapshot-max-cells 800] [--model-snapshot-max-cell-chars 256] [--model-repair-attempts 1] [--refresh-excel-caches] [--task-ids-file ] [--offset 0] [--limit 3] [--repeats 5] [--retry-failed 2] [--retry-score-failures] [--compare-charts] [--clean] [--json-out ]", "", "copy-input-baseline proves runner/export/scoring plumbing.", "apply-agent-patch reads agent/edit-plan.json, edits the workbook, emits a candidate, then opens evaluator metadata.", @@ -67,6 +68,9 @@ if (modelBatchSize < 1 || modelBatchSize > 16) { if (mode === "nodeagent-workbook" && modelBatchSize > 1) { throw new Error("nodeagent-workbook requires --model-batch-size 1."); } +if (refreshExcelCaches && mode !== "nodeagent-workbook") { + throw new Error("--refresh-excel-caches requires --mode nodeagent-workbook."); +} if (modelSnapshotMaxCells !== undefined && modelSnapshotMaxCells < 1) { throw new Error("--model-snapshot-max-cells must be at least 1."); } @@ -94,6 +98,7 @@ const report = await runStagedSpreadsheetBench({ modelSnapshotMaxCells, modelSnapshotMaxCellChars, modelRepairAttempts, + refreshExcelCaches, taskIds, limit, offset, diff --git a/src/eval/spreadsheetBenchNodeAgentBridge.ts b/src/eval/spreadsheetBenchNodeAgentBridge.ts index aa0102bf..ede85fb2 100644 --- a/src/eval/spreadsheetBenchNodeAgentBridge.ts +++ b/src/eval/spreadsheetBenchNodeAgentBridge.ts @@ -140,6 +140,20 @@ export type SpreadsheetBenchNodeAgentRecalculationReceipt = { }>; }; +export type SpreadsheetBenchCandidateFinalizationReceipt = { + engine: string; + beforeSha256: string; + afterSha256: string; + changed: boolean; + formulaCellCount?: number; + cacheWriteMode?: string; + receipt: { + path: string; + sha256: string; + bytes: number; + }; +}; + export type SpreadsheetBenchNodeAgentBridgeReceipt = { schema: typeof SPREADSHEETBENCH_NODEAGENT_BRIDGE_SCHEMA; traceId: string; @@ -169,6 +183,7 @@ export type SpreadsheetBenchNodeAgentBridgeReceipt = { usage: TokenUsage; }; recalculation: SpreadsheetBenchNodeAgentRecalculationReceipt; + candidateFinalization?: SpreadsheetBenchCandidateFinalizationReceipt; frame: ReasoningFrameRunReceipt; trace: NodeAgentTrace; }; @@ -185,6 +200,14 @@ export type RunSpreadsheetBenchNodeAgentBridgeOptions = { snapshotMaxCells?: number; scanMaxCells?: number; now?: () => number; + finalizeCandidate?: (args: { + taskId: string; + track: SpreadsheetBenchTrack; + category?: string; + sourceWorkbookPath: string; + candidateWorkbookPath: string; + beforeSha256: string; + }) => SpreadsheetBenchCandidateFinalizationReceipt | Promise; }; type OpenedAgentTask = { @@ -263,7 +286,29 @@ export async function runSpreadsheetBenchNodeAgentBridge( candidateWorkbookPath, patches: room.packageMutations(), }); + const emittedCandidateSha256 = sha256File(candidateWorkbookPath); + const candidateFinalization = options.finalizeCandidate + ? await options.finalizeCandidate({ + taskId: task.manifest.taskId, + track: task.manifest.track, + ...(task.manifest.category ? { category: task.manifest.category } : {}), + sourceWorkbookPath: task.sourceWorkbookPath, + candidateWorkbookPath, + beforeSha256: emittedCandidateSha256, + }) + : undefined; const candidateWorkbookSha256 = sha256File(candidateWorkbookPath); + if (candidateFinalization) { + if (candidateFinalization.beforeSha256 !== emittedCandidateSha256) { + throw new Error("SpreadsheetBench candidate finalizer beforeSha256 does not match the emitted workbook"); + } + if (candidateFinalization.afterSha256 !== candidateWorkbookSha256) { + throw new Error("SpreadsheetBench candidate finalizer afterSha256 does not match the finalized workbook"); + } + if (candidateFinalization.changed !== (emittedCandidateSha256 !== candidateWorkbookSha256)) { + throw new Error("SpreadsheetBench candidate finalizer changed flag does not match the workbook hashes"); + } + } const stages = buildStageReceipts(traceId, frameReceipt); const mutatingTask = room.taskInspection().mutatingTask; const outcome = bridgeOutcome( @@ -287,6 +332,7 @@ export async function runSpreadsheetBenchNodeAgentBridge( artifactIds: room.artifactIds(), outcomeStatus: outcome.status, recalculation, + candidateFinalization, }); return { @@ -319,6 +365,7 @@ export async function runSpreadsheetBenchNodeAgentBridge( }, }, recalculation, + ...(candidateFinalization ? { candidateFinalization } : {}), frame: frameReceipt, trace, }; @@ -1707,6 +1754,7 @@ function buildBridgeTrace(args: { artifactIds: string[]; outcomeStatus: SpreadsheetBenchNodeAgentBridgeReceipt["outcome"]["status"]; recalculation: SpreadsheetBenchNodeAgentRecalculationReceipt; + candidateFinalization?: SpreadsheetBenchCandidateFinalizationReceipt; }): NodeAgentTrace { const candidateRef = traceRef("artifact", args.candidateWorkbookPath, { label: `SpreadsheetBench candidate ${basename(args.candidateWorkbookPath)}`, @@ -1759,6 +1807,20 @@ function buildBridgeTrace(args: { verifier: args.recalculation.engine, status: args.recalculation.unresolvedFormulaCount === 0 ? "verified" : "needs_review", })); + if (args.candidateFinalization) { + trace.evidence.push(makeEvidenceReceipt({ + traceId: args.traceId, + label: "SpreadsheetBench candidate finalization", + sourceRefs: [traceRef("tool_result", `${args.traceId}:candidate-finalization`, { + label: args.candidateFinalization.engine, + hash: stableTraceHash(args.candidateFinalization), + })], + artifactRefs: [candidateRef], + fact: args.candidateFinalization, + verifier: args.candidateFinalization.engine, + status: "verified", + })); + } trace.mutations = args.frameReceipt.agentResult.trace.flatMap((event): MutationReceipt[] => { if (!MUTATION_TOOL_NAMES.has(event.tool)) return []; const targets = writeTargets(event.args, event.result).map((target) => traceRef("cell", target, { label: "SpreadsheetBench workbook target" })); diff --git a/src/eval/spreadsheetBenchRunner.ts b/src/eval/spreadsheetBenchRunner.ts index 9d39c3e8..b707ec99 100644 --- a/src/eval/spreadsheetBenchRunner.ts +++ b/src/eval/spreadsheetBenchRunner.ts @@ -12,7 +12,10 @@ import type { SpreadsheetBenchTrack } from "./spreadsheetBenchAdapter"; import type { AgentModel, TokenUsage } from "../nodeagent/core/types"; import { priceRun } from "../nodeagent/models/adapter"; import { getModelPricing } from "../nodeagent/models/modelCatalog"; -import { runSpreadsheetBenchNodeAgentBridge } from "./spreadsheetBenchNodeAgentBridge"; +import { + runSpreadsheetBenchNodeAgentBridge, + type SpreadsheetBenchCandidateFinalizationReceipt, +} from "./spreadsheetBenchNodeAgentBridge"; import { checksForWorkbookOperations, extractWorkbookTaskReferences, @@ -86,6 +89,7 @@ export type SpreadsheetBenchRunnerOptions = { modelSnapshotMaxCells?: number; modelSnapshotMaxCellChars?: number; modelRepairAttempts?: number; + refreshExcelCaches?: boolean; taskIds?: string[]; repeats?: number; retryFailed?: number; @@ -174,6 +178,7 @@ export type SpreadsheetBenchSidecarEvidence = { editVerification?: SpreadsheetBenchSidecarFileEvidence; nodeAgentReceipt?: SpreadsheetBenchSidecarFileEvidence; nodeAgentTrace?: SpreadsheetBenchSidecarFileEvidence; + candidateFinalization?: SpreadsheetBenchSidecarFileEvidence; repairOutputs?: SpreadsheetBenchSidecarFileEvidence[]; formulaResultPolicy?: string; supportedFormulaFunctions?: string[]; @@ -227,6 +232,7 @@ export type SpreadsheetBenchRunnerReport = { snapshotMaxCellChars: number | null; instructionMaxChars: number | null; repairAttempts: number; + refreshExcelCaches?: boolean; selectedTaskCount: number; }; budget: { @@ -467,6 +473,7 @@ export async function runStagedSpreadsheetBench(options: SpreadsheetBenchRunnerO : Math.max(1, Math.trunc(options.modelSnapshotMaxCellChars)), instructionMaxChars: (options.modelBatchSize ?? 1) > 1 ? BATCH_INSTRUCTION_MAX_CHARS : null, repairAttempts: Math.max(0, Math.min(3, Math.trunc(options.modelRepairAttempts ?? 0))), + refreshExcelCaches: options.refreshExcelCaches === true, selectedTaskCount: selectedTasks.length, }, budget: { @@ -536,6 +543,7 @@ async function runTask( modelSnapshotMaxCells: options.modelSnapshotMaxCells, modelSnapshotMaxCellChars: options.modelSnapshotMaxCellChars, modelRepairAttempts: options.modelRepairAttempts, + refreshExcelCaches: options.refreshExcelCaches, batchedModelPlan, }); let emitted: string | ModelCandidateEmission; @@ -721,6 +729,7 @@ function collectSidecarEvidence(outputRoot: string, taskOutDir: string): Spreads editVerification?: string; nodeAgentReceipt?: string; nodeAgentTrace?: string; + candidateFinalization?: string; repairOutputs?: string[]; formulaResultPolicy?: string; supportedFormulaFunctions?: string[]; @@ -745,6 +754,9 @@ function collectSidecarEvidence(outputRoot: string, taskOutDir: string): Spreads ...(manifest.editVerification ? { editVerification: fileEvidence(outputRoot, resolveSidecarPath(taskOutDir, manifest.editVerification)) } : {}), ...(manifest.nodeAgentReceipt ? { nodeAgentReceipt: fileEvidence(outputRoot, resolveSidecarPath(taskOutDir, manifest.nodeAgentReceipt)) } : {}), ...(manifest.nodeAgentTrace ? { nodeAgentTrace: fileEvidence(outputRoot, resolveSidecarPath(taskOutDir, manifest.nodeAgentTrace)) } : {}), + ...(manifest.candidateFinalization + ? { candidateFinalization: fileEvidence(outputRoot, resolveSidecarPath(taskOutDir, manifest.candidateFinalization)) } + : {}), ...(manifest.repairOutputs?.length ? { repairOutputs: manifest.repairOutputs.map((path) => fileEvidence(outputRoot, resolveSidecarPath(taskOutDir, path))) } : {}), @@ -783,6 +795,7 @@ function emitCandidateWorkbook(args: { modelSnapshotMaxCells?: number; modelSnapshotMaxCellChars?: number; modelRepairAttempts?: number; + refreshExcelCaches?: boolean; batchedModelPlan?: BatchedModelPlan; }): Promise | string { const firstInput = args.agent.inputFiles[0]; @@ -841,6 +854,7 @@ async function emitNodeAgentWorkbookCandidate(args: { modelTimeoutMs?: number; modelSnapshotMaxCells?: number; modelRepairAttempts?: number; + refreshExcelCaches?: boolean; }): Promise { if (!args.model) throw new Error(`nodeagent-workbook requires options.model: ${args.agent.taskId}`); const started = Date.now(); @@ -855,6 +869,18 @@ async function emitNodeAgentWorkbookCandidate(args: { maxSteps, modelTimeoutMs: runTimeoutMs, snapshotMaxCells: args.modelSnapshotMaxCells, + ...(args.refreshExcelCaches + ? { + finalizeCandidate: ({ candidateWorkbookPath, beforeSha256 }: { + candidateWorkbookPath: string; + beforeSha256: string; + }) => refreshCandidateExcelCaches({ + candidateWorkbookPath, + beforeSha256, + receiptPath: join(args.taskOutDir, "candidate-finalization.json"), + }), + } + : {}), }); const modelPlanningMs = Date.now() - started; const receiptPath = join(args.taskOutDir, "nodeagent-workbook-receipt.json"); @@ -888,6 +914,9 @@ async function emitNodeAgentWorkbookCandidate(args: { candidateWorkbook: basename(args.target), nodeAgentReceipt: basename(receiptPath), nodeAgentTrace: basename(tracePath), + ...(receipt.candidateFinalization + ? { candidateFinalization: basename(receipt.candidateFinalization.receipt.path) } + : {}), appliedOperationCount: receipt.outcome.changedCellCount, repairAttemptCount: receipt.stages.repair.attempts, verificationStatus: receipt.outcome.finalVerificationStatus === "passed" ? "passed" : "needs_repair", @@ -905,6 +934,63 @@ async function emitNodeAgentWorkbookCandidate(args: { return { path: args.target, modelPlanningMs, model: modelInfo }; } +function refreshCandidateExcelCaches(args: { + candidateWorkbookPath: string; + beforeSha256: string; + receiptPath: string; +}): SpreadsheetBenchCandidateFinalizationReceipt { + const script = resolve("scripts", "spreadsheetbench-refresh-excel.py"); + const result = spawnSync( + process.env.PYTHON?.trim() || "python", + [script, "--file", args.candidateWorkbookPath, "--receipt", args.receiptPath], + { + cwd: process.cwd(), + encoding: "utf8", + windowsHide: true, + timeout: 10 * 60_000, + maxBuffer: 4 * 1024 * 1024, + }, + ); + if (result.error) { + throw new Error(`SpreadsheetBench Excel cache refresh failed to start: ${result.error.message}`); + } + if (result.status !== 0) { + const detail = [result.stderr, result.stdout].filter(Boolean).join(" ").replace(/\s+/g, " ").trim().slice(0, 1_000); + throw new Error(`SpreadsheetBench Excel cache refresh failed (${String(result.status)}): ${detail}`); + } + const refresh = readJson<{ + engine?: string; + records?: Array<{ + status?: string; + beforeSha256?: string; + afterSha256?: string; + formulaCellCount?: number; + cacheWriteMode?: string; + }>; + }>(args.receiptPath); + const record = refresh.records?.[0]; + if (!record || record.status !== "refreshed") { + throw new Error("SpreadsheetBench Excel cache refresh receipt is missing a refreshed record"); + } + if (record.beforeSha256 !== args.beforeSha256 || !record.afterSha256) { + throw new Error("SpreadsheetBench Excel cache refresh receipt hash boundary is invalid"); + } + const receiptContent = readFileSync(args.receiptPath); + return { + engine: refresh.engine?.trim() || "spreadsheetbench-excel-cache-refresh", + beforeSha256: record.beforeSha256, + afterSha256: record.afterSha256, + changed: record.beforeSha256 !== record.afterSha256, + ...(typeof record.formulaCellCount === "number" ? { formulaCellCount: record.formulaCellCount } : {}), + ...(record.cacheWriteMode ? { cacheWriteMode: record.cacheWriteMode } : {}), + receipt: { + path: resolve(args.receiptPath), + sha256: createHash("sha256").update(receiptContent).digest("hex"), + bytes: receiptContent.byteLength, + }, + }; +} + type AgentWorkspace = { root: string; agentDir: string; diff --git a/tests/spreadsheetBenchNodeAgentBridge.test.ts b/tests/spreadsheetBenchNodeAgentBridge.test.ts index 1183fed7..b91a5094 100644 --- a/tests/spreadsheetBenchNodeAgentBridge.test.ts +++ b/tests/spreadsheetBenchNodeAgentBridge.test.ts @@ -1,3 +1,4 @@ +import { createHash } from "node:crypto"; import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -11,6 +12,10 @@ import type { AgentMessage, AgentModel, AgentStep } from "../src/nodeagent/core/ const roots: string[] = []; +function sha256(value: Buffer): string { + return createHash("sha256").update(value).digest("hex"); +} + afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); @@ -197,6 +202,63 @@ describe("SpreadsheetBench canonical NodeAgent bridge", () => { ); }); + it("finalizes the emitted workbook before sealing candidate and trace hashes", async () => { + const root = tempRoot(); + const task = await stagedTask(root); + const candidate = join(root, "output", "finalized.xlsx"); + const finalizationPath = join(root, "output", "candidate-finalization.json"); + const capture = { systems: [] as string[], messages: [] as string[], toolNames: [] as string[][] }; + + const receipt = await runSpreadsheetBenchNodeAgentBridge({ + agentManifestPath: task.agentManifest, + candidateWorkbookPath: candidate, + model: repairingWorkbookModel(capture), + traceId: "trace_sbench_candidate_finalization", + maxSteps: 10, + now: () => 1_125, + finalizeCandidate: async ({ candidateWorkbookPath, beforeSha256 }) => { + const before = readFileSync(candidateWorkbookPath); + expect(sha256(before)).toBe(beforeSha256); + const zip = await JSZip.loadAsync(before); + expect(await zip.file("xl/worksheets/sheet1.xml")?.async("string")).toContain("A1*2"); + zip.file("customXml/finalized.txt", "cache refresh complete"); + const finalized = await zip.generateAsync({ type: "nodebuffer" }); + writeFileSync(candidateWorkbookPath, finalized); + writeFileSync(finalizationPath, '{"schema":1,"status":"refreshed"}\n'); + const finalization = readFileSync(finalizationPath); + return { + engine: "test-cache-refresh", + beforeSha256, + afterSha256: sha256(finalized), + changed: true, + formulaCellCount: 1, + cacheWriteMode: "test", + receipt: { + path: finalizationPath, + sha256: sha256(finalization), + bytes: finalization.byteLength, + }, + }; + }, + }); + + expect(receipt.candidateWorkbookSha256).toBe(sha256(readFileSync(candidate))); + expect(receipt.candidateFinalization).toMatchObject({ + engine: "test-cache-refresh", + changed: true, + beforeSha256: expect.stringMatching(/^[a-f0-9]{64}$/), + afterSha256: receipt.candidateWorkbookSha256, + receipt: { path: finalizationPath }, + }); + expect(receipt.trace.evidence).toContainEqual(expect.objectContaining({ + label: "SpreadsheetBench candidate finalization", + status: "verified", + })); + expect(JSON.stringify(receipt.trace.eval.proofArtifacts)).toContain(receipt.candidateWorkbookSha256); + const finalizedZip = await JSZip.loadAsync(readFileSync(candidate)); + expect(await finalizedZip.file("customXml/finalized.txt")?.async("string")).toBe("cache refresh complete"); + }); + it("treats invalid Excel dates as inspectable values instead of aborting the trace", async () => { const root = tempRoot(); const task = await stagedTask(root, "Inspect the workbook and report what remains."); diff --git a/tests/spreadsheetbench_refresh_excel_test.py b/tests/spreadsheetbench_refresh_excel_test.py index 3f689edd..0260e519 100644 --- a/tests/spreadsheetbench_refresh_excel_test.py +++ b/tests/spreadsheetbench_refresh_excel_test.py @@ -288,5 +288,39 @@ def test_rejects_stale_expected_sha_without_modifying_the_workbook(self) -> None self.assertEqual([path.name for path in Path(temp_dir).iterdir()], ["book.xlsx"]) +class WorkbookSelectionTests(unittest.TestCase): + def test_selects_one_explicit_candidate_without_output_suffix(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + candidate = root / "candidate-input.xlsx" + candidate.write_bytes(b"xlsx") + + selected_root, files = refresh_excel.select_workbooks(None, str(candidate)) + + self.assertEqual(selected_root, candidate.parent.resolve()) + self.assertEqual(files, [candidate.resolve()]) + + def test_directory_mode_keeps_the_official_output_filename_contract(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + expected = root / "nested" / "01_output.xlsx" + expected.parent.mkdir() + expected.write_bytes(b"xlsx") + (root / "candidate.xlsx").write_bytes(b"xlsx") + + selected_root, files = refresh_excel.select_workbooks(str(root), None) + + self.assertEqual(selected_root, root.resolve()) + self.assertEqual(files, [expected.resolve()]) + + def test_rejects_ambiguous_or_invalid_sources(self) -> None: + with self.assertRaisesRegex(ValueError, "exactly one"): + refresh_excel.select_workbooks(None, None) + with self.assertRaisesRegex(ValueError, "exactly one"): + refresh_excel.select_workbooks(".", "candidate.xlsx") + with self.assertRaisesRegex(ValueError, "existing .xlsx"): + refresh_excel.select_workbooks(None, "missing.xlsx") + + if __name__ == "__main__": unittest.main() From e83b9843261b2d9299f144924041acd61a119b96 Mon Sep 17 00:00:00 2001 From: homen Date: Thu, 16 Jul 2026 11:52:41 -0700 Subject: [PATCH 11/12] feat: harden NodeAgent workbook and workspace flows Areas: .env, convex, docs, e2e, scripts, src, tests. --- .env.example | 4 + convex/agent.ts | 2 + convex/agentJobs.ts | 106 +- convex/artifacts.ts | 75 +- convex/messages.ts | 3 +- convex/schema.ts | 4 + convex/streaming.ts | 3 +- docs/design/COMPONENT_MAP.md | 3 +- docs/design/UI_CONTRACT.md | 34 +- docs/nodeagent/AGENT_READY_API.md | 38 +- e2e/deployed-auth-first-user.spec.ts | 2 +- e2e/mobile-story-surfaces.spec.ts | 2 +- e2e/trace-tab.spec.ts | 20 +- scripts/openrouter-free-discover.ts | 84 +- .../spreadsheetbench-accept-official-v2.ts | 3 +- .../spreadsheetbench-official-v2-project.ts | 38 +- scripts/spreadsheetbench-refresh-excel.py | 1011 +++++-- scripts/spreadsheetbench-run-chunked.ts | 58 +- scripts/spreadsheetbench-run.ts | 7 +- .../spreadsheetbench-structural-repair.ps1 | 126 + src/app/store.tsx | 92 +- src/app/styles.css | 18 +- src/engine/roomEngine.ts | 183 ++ src/engine/types.ts | 4 + src/eval/spreadsheetBenchNodeAgentBridge.ts | 825 +++++- .../spreadsheetBenchOfficialV2Acceptance.ts | 14 + src/eval/spreadsheetBenchReportRepair.ts | 9 +- .../spreadsheetBenchRunSourceFingerprint.ts | 59 + src/eval/spreadsheetBenchRunner.ts | 390 ++- src/eval/spreadsheetBenchStructuralRepair.ts | 221 ++ src/eval/spreadsheetBenchWorkbookEmitter.ts | 155 +- src/nodeagent/core/runtime.ts | 72 +- src/nodeagent/guardrails/workbookWorkflow.ts | 20 +- src/nodeagent/models/adapter.ts | 2 +- src/nodeagent/models/convexModel.ts | 12 +- .../skills/spreadsheet/cellMutator.ts | 483 +++- .../spreadsheet/workbookTaskIntelligence.ts | 2328 ++++++++++++++++- src/shared/spreadsheetFontColor.ts | 124 + src/ui/Chat.tsx | 97 +- src/ui/RoomShell.tsx | 147 +- src/ui/mobile/MobileApp.tsx | 4 +- src/ui/panels/Artifact.tsx | 121 +- src/ui/panels/notebook-paper.css | 190 ++ src/ui/primitives/primitives.css | 33 +- .../workArtifacts/DeckStoryboardWorkbench.tsx | 10 +- src/ui/workArtifacts/ProposalReviewCenter.tsx | 7 +- src/ui/workArtifacts/WorkArtifactsPanel.tsx | 152 +- src/ui/workArtifacts/collaborativeDeck.ts | 52 + src/ui/workArtifacts/deckObjectModel.ts | 42 + src/ui/workArtifacts/deckPptxExport.ts | 5 +- src/ui/workArtifacts/notebookStructure.ts | 84 +- src/ui/workArtifacts/work-artifacts.css | 75 + .../workArtifacts/workArtifactsNavigation.ts | 15 + tests/agentJobsRuntime.test.ts | 57 + tests/agentRuntime.test.ts | 93 + tests/artifactEditBundle.test.ts | 173 ++ tests/artifactXlsxExport.test.ts | 61 +- tests/artifactsPseudoTabPersistence.test.tsx | 88 + tests/chatScale.test.ts | 34 + tests/deckBinderRouting.test.tsx | 142 + tests/deckCollaborationUi.test.tsx | 25 +- tests/deckObjectMerge.test.ts | 114 + .../modelAdapterProviderNeutralRoute.test.ts | 6 +- tests/nodeagentProviderToolSchema.test.ts | 8 +- tests/notebookPaper.test.tsx | 22 +- tests/roomEngineAtomicEdits.test.ts | 106 + tests/roomShellLayout.test.tsx | 106 + tests/spreadsheetBenchChunkedRepair.test.ts | 34 + tests/spreadsheetBenchNodeAgentBridge.test.ts | 387 ++- ...readsheetBenchOfficialV2Acceptance.test.ts | 19 + .../spreadsheetBenchOfficialV2Project.test.ts | 171 +- tests/spreadsheetBenchReportRepair.test.ts | 15 +- ...spreadsheetBenchRunnerFinalization.test.ts | 280 ++ .../spreadsheetBenchStructuralRepair.test.ts | 75 + tests/spreadsheetBenchWorkbookEmitter.test.ts | 111 +- tests/spreadsheetFontColor.test.ts | 57 + tests/spreadsheetFontColorMutations.test.ts | 107 + tests/spreadsheetbench_refresh_excel_test.py | 570 +++- tests/workArtifacts.test.ts | 34 + tests/workArtifactsNavigation.test.ts | 21 + .../workArtifactsNotebookReadModelUi.test.tsx | 82 + tests/workbookAgentTools.test.ts | 17 +- tests/workbookTaskIntelligence.test.ts | 681 +++++ tests/workbookWorkflowHook.test.ts | 65 + 84 files changed, 10780 insertions(+), 649 deletions(-) create mode 100644 scripts/spreadsheetbench-structural-repair.ps1 create mode 100644 src/eval/spreadsheetBenchOfficialV2Acceptance.ts create mode 100644 src/eval/spreadsheetBenchRunSourceFingerprint.ts create mode 100644 src/eval/spreadsheetBenchStructuralRepair.ts create mode 100644 src/shared/spreadsheetFontColor.ts create mode 100644 src/ui/workArtifacts/workArtifactsNavigation.ts create mode 100644 tests/artifactEditBundle.test.ts create mode 100644 tests/artifactsPseudoTabPersistence.test.tsx create mode 100644 tests/deckBinderRouting.test.tsx create mode 100644 tests/deckObjectMerge.test.ts create mode 100644 tests/roomEngineAtomicEdits.test.ts create mode 100644 tests/roomShellLayout.test.tsx create mode 100644 tests/spreadsheetBenchOfficialV2Acceptance.test.ts create mode 100644 tests/spreadsheetBenchRunnerFinalization.test.ts create mode 100644 tests/spreadsheetBenchStructuralRepair.test.ts create mode 100644 tests/spreadsheetFontColor.test.ts create mode 100644 tests/spreadsheetFontColorMutations.test.ts create mode 100644 tests/workArtifactsNavigation.test.ts create mode 100644 tests/workArtifactsNotebookReadModelUi.test.tsx diff --git a/.env.example b/.env.example index 17516ac2..a2d9b474 100644 --- a/.env.example +++ b/.env.example @@ -52,6 +52,10 @@ OPENROUTER_FREE_AUTO_LIMIT=8 OPENROUTER_FREE_MODEL_CACHE_MS=600000 OPENROUTER_REQUIRE_NO_TRAINING=1 OPENROUTER_FREE_ALLOW_FILE_EGRESS=0 +# Provider-neutral NodeAgent uses OpenRouter's governed free ladder first, then +# this direct Gemini route when free routes are unavailable. Receipts retain the +# catalog price because an account's free-tier entitlement cannot be asserted. +NODEAGENT_FREE_AUTO_GOOGLE_MODEL=gemini-3.5-flash AGENT_MODEL=gpt-5.4-nano # Optional voice provider overrides. Browser TTS is the free client default; # Convex `/voice/*` endpoints are authenticated hosted fallbacks. diff --git a/convex/agent.ts b/convex/agent.ts index d59122cf..4056d8a6 100644 --- a/convex/agent.ts +++ b/convex/agent.ts @@ -302,6 +302,7 @@ export const runRoomAgent = action({ text: finalText.slice(0, 4_000), clientMsgId: `plan-blocked-${String(jobClaim.jobId)}`, kind: "agent", + jobId: jobClaim.jobId, }); // Release the credit hold immediately — this run was blocked before any spend (no run yet). // (Other early exits before the run, e.g. egress-blocked, are reclaimed by the sweep cron.) @@ -733,6 +734,7 @@ export const runRoomAgent = action({ text: visibleFallback.slice(0, 4_000), clientMsgId: `final-${String(runId)}`, kind: "agent", + jobId, }); } return { diff --git a/convex/agentJobs.ts b/convex/agentJobs.ts index 62d3d51f..aecbe4ef 100644 --- a/convex/agentJobs.ts +++ b/convex/agentJobs.ts @@ -1011,6 +1011,7 @@ async function ensureTerminalJobReceipts(ctx: any, args: { jobId: args.job._id, text, }); + await activateNextQueuedJob(ctx, args.job._id, args.now); return text; } @@ -1069,6 +1070,63 @@ async function terminalizeAgentJob(ctx: any, args: { return { ok: true as const }; } +async function activateNextQueuedJob(ctx: any, predecessorJobId: Id<"agentJobs">, now: number): Promise { + const predecessor = await ctx.db.get(predecessorJobId); + if (!predecessor || !terminalStatuses.has(predecessor.status)) return; + const successors = await ctx.db.query("agentJobs") + .withIndex("by_waiting_for", (q: any) => q.eq("waitingForJobId", predecessorJobId)) + .order("asc") + .take(5); + const next = successors.find((job: any) => job.status === "queued" && !job.workflowId); + if (!next) return; + + if (predecessor.waitingForJobId) { + const earlier = await ctx.db.get(predecessor.waitingForJobId); + if (earlier && !terminalStatuses.has(earlier.status)) { + await ctx.db.patch(next._id, { waitingForJobId: earlier._id, updatedAt: now }); + return; + } + } + + try { + const workflowId = String(await startWorkflow(ctx, internal.agentWorkflows.freeAutoWorkflow, { jobId: next._id }, { + onComplete: internal.agentWorkflows.freeAutoWorkflowComplete, + context: { jobId: next._id }, + })); + await ctx.db.patch(next._id, { + workflowId, + waitingForJobId: undefined, + nextRunAt: now, + schedulerHandoffCount: (next.schedulerHandoffCount ?? 0) + 1, + updatedAt: now, + }); + await recordOperationEvent(ctx, { + jobId: next._id, + sequence: 2, + kind: "scheduler", + name: "agentJobs.activateQueuedSuccessor", + targetKind: "artifact", + targetId: String(next.artifactId), + status: "completed", + countDelta: 1, + affectedIds: [String(predecessorJobId), String(next._id)], + startedAt: now, + completedAt: now, + }); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + const safeMessage = `queued_workflow_start_failed: ${message || "unknown"}`.slice(0, 1_000); + await ctx.db.patch(next._id, { + status: "failed", + waitingForJobId: undefined, + error: safeMessage, + completedAt: now, + updatedAt: now, + }); + await ensureTerminalJobReceipts(ctx, { job: next, status: "failed", error: safeMessage, now }); + } +} + export const recordStreamEvent = internalMutation({ args: { jobId: v.id("agentJobs"), @@ -1888,6 +1946,7 @@ type DurableStartAgentJobArgs = { planPreview?: unknown; error?: string; operationName?: string; + deferUntilJobId?: Id<"agentJobs">; }; type DurableStartAgentJobResult = { jobId: Id<"agentJobs">; @@ -2272,6 +2331,7 @@ async function startDurableAgentJob(ctx: any, a: DurableStartAgentJobArgs): Prom scope, commandText: a.goal, request, + waitingForJobId: a.deferUntilJobId, priority: 0, approvalPolicy, evidencePolicy, @@ -2294,9 +2354,9 @@ async function startDurableAgentJob(ctx: any, a: DurableStartAgentJobArgs): Prom mutationCount: 1, modelCallCount: 0, toolCallCount: 0, - schedulerHandoffCount: execution === "workflow" && !planBlocked ? 1 : 0, + schedulerHandoffCount: execution === "workflow" && !planBlocked && !a.deferUntilJobId ? 1 : 0, receiptCount: 0, - nextRunAt: now, + nextRunAt: a.deferUntilJobId ? 0 : now, createdAt: now, updatedAt: now, completedAt: status === "blocked" ? now : undefined, @@ -2320,7 +2380,7 @@ async function startDurableAgentJob(ctx: any, a: DurableStartAgentJobArgs): Prom status: "started", title: "Room NodeAgent", text: a.goal, - metadata: { entrypoint, scope, routePolicy, runtimePolicy, runtimeProfile, modelPolicy, fileEgressPromoted: promotedForFileEgress || undefined, freeFileEgressPromotionBlocked: freeFileEgressBlocked && !promotedForFileEgress || undefined }, + metadata: { entrypoint, scope, routePolicy, runtimePolicy, runtimeProfile, modelPolicy, waitingForJobId: a.deferUntilJobId ? String(a.deferUntilJobId) : undefined, fileEgressPromoted: promotedForFileEgress || undefined, freeFileEgressPromotionBlocked: freeFileEgressBlocked && !promotedForFileEgress || undefined }, createdAt: now, }); if (status === "blocked") { @@ -2379,6 +2439,9 @@ async function startDurableAgentJob(ctx: any, a: DurableStartAgentJobArgs): Prom updatedAt: now, })); } + if (a.deferUntilJobId) { + return { jobId, reused: false as const, status: "queued" as const, modelPolicy, routePolicy, runtimePolicy }; + } await recordOperationEvent(ctx, { jobId, sequence: 2, @@ -2468,8 +2531,37 @@ export const startPublicAsk = mutation({ modelPolicy: v.optional(v.string()), runtimeProfile: v.optional(runtimeProfileV), maxAttempts: v.optional(v.number()), + disposition: v.optional(v.union(v.literal("start"), v.literal("queue"), v.literal("redirect"))), }, handler: async (ctx, a): Promise => { + const requester = await requireActorProof(ctx, a.roomId, a.requester); + const recentJobs = await ctx.db.query("agentJobs") + .withIndex("by_room", (q: any) => q.eq("roomId", a.roomId)) + .order("desc") + .take(50); + const activeJobs = recentJobs.filter((job: any) => ( + job.requester?.id === requester.id + && (job.scope ?? "public_room") === "public_room" + && !terminalStatuses.has(job.status) + )); + const disposition = a.disposition ?? "start"; + if (disposition === "redirect") { + for (const job of activeJobs) { + if (job.workflowId) { + try { + await cancelWorkflow(ctx, components.workflow, job.workflowId as never); + } catch { + // The job row and lease fence remain authoritative if the workflow settled first. + } + } + await terminalizeAgentJob(ctx, { + job, + status: "cancelled", + error: "redirected_by_followup", + operationName: "agentJobs.redirect", + }); + } + } const artifact = await resolvePublicAskArtifact(ctx, a); const allowedElementIds = normalizePublicAskAllowedElementIds(a.allowedElementIds); if (allowedElementIds?.length) { @@ -2486,6 +2578,12 @@ export const startPublicAsk = mutation({ modelPolicy: a.modelPolicy, runtimeProfile: a.runtimeProfile, maxAttempts: a.maxAttempts, + ...(disposition === "queue" && activeJobs[0] ? { + deferUntilJobId: activeJobs[0]._id as Id<"agentJobs">, + idempotencyKey: `public-ask-queue:${String(a.roomId)}:${requester.id}:${String(activeJobs[0]._id)}:${Date.now()}`, + } : disposition === "redirect" ? { + idempotencyKey: `public-ask-redirect:${String(a.roomId)}:${requester.id}:${Date.now()}`, + } : {}), mode: modeForArtifact(artifact) ?? (artifact.kind === "note" ? undefined : goalPrefersPersonResearch(a.goal) || goalPrefersCompanyResearch(a.goal) ? "research" : undefined), }); return startDurableAgentJob(ctx, { @@ -2500,6 +2598,8 @@ export const startPublicAsk = mutation({ allowedElementIds, mutationScope: allowedElementIds?.length ? "element_allowlist" : undefined, source: "public_chat", + disposition, + waitingForJobId: disposition === "queue" && activeJobs[0] ? String(activeJobs[0]._id) : undefined, runtimeProfile: a.runtimeProfile, }, }); diff --git a/convex/artifacts.ts b/convex/artifacts.ts index 2efa8e72..2d15e180 100644 --- a/convex/artifacts.ts +++ b/convex/artifacts.ts @@ -10,7 +10,7 @@ * re-reads and retries. Same function backs hand-edits from the UI. */ -import { v } from "convex/values"; +import { ConvexError, v } from "convex/values"; import { internal } from "./_generated/api"; import { internalMutation, internalQuery, mutation, query } from "./_generated/server"; import type { MutationCtx } from "./_generated/server"; @@ -32,6 +32,7 @@ const MAX_ARTIFACT_TITLE_CHARS = 180; const MAX_ARTIFACT_SEED_ELEMENTS = 8_192; const MAX_ARTIFACT_SEED_BYTES = 5_000_000; const MAX_ELEMENT_ID_CHARS = 160; +const MAX_ARTIFACT_EDIT_BUNDLE_EDITS = 64; const MAX_RAW_UPLOAD_BYTES = 25_000_000; const MAX_UPLOAD_FILE_NAME_CHARS = 240; const MAX_UPLOAD_MIME_CHARS = 200; @@ -1088,6 +1089,78 @@ export const applyCellEdit = mutation({ handler: async (ctx, a) => applyCellEditCore(ctx, { ...a, actor: await requireActorProof(ctx, a.roomId, a.proof) }), }); +/** Atomic UI edit bundle: every element CAS succeeds, or the transaction writes nothing. */ +export const applyArtifactEdits = mutation({ + args: { + roomId: v.id("rooms"), + artifactId: v.id("artifacts"), + edits: v.array(v.object({ + opId: v.string(), + elementId: v.string(), + kind: v.union(v.literal("set"), v.literal("create"), v.literal("delete")), + value: v.any(), + baseVersion: v.number(), + })), + requester: actorProofV, + }, + handler: async (ctx, a) => { + const actor = await requireActorProof(ctx, a.roomId, a.requester); + if (a.edits.length === 0 || a.edits.length > MAX_ARTIFACT_EDIT_BUNDLE_EDITS) { + throw new ConvexError({ + code: "invalid_artifact_edit_bundle", + reason: "edit_count_out_of_bounds", + minimum: 1, + maximum: MAX_ARTIFACT_EDIT_BUNDLE_EDITS, + actual: a.edits.length, + }); + } + + const opIds = new Set(); + const elementIds = new Set(); + for (const edit of a.edits) { + if (opIds.has(edit.opId)) { + throw new ConvexError({ code: "invalid_artifact_edit_bundle", reason: "duplicate_op_id", opId: edit.opId }); + } + if (elementIds.has(edit.elementId)) { + throw new ConvexError({ code: "invalid_artifact_edit_bundle", reason: "duplicate_element_id", elementId: edit.elementId }); + } + opIds.add(edit.opId); + elementIds.add(edit.elementId); + } + + const results: Array<{ opId: string; elementId: string; version: number }> = []; + for (const edit of a.edits) { + const result = await applyCellEditCore(ctx, { + roomId: a.roomId, + artifactId: a.artifactId, + elementId: edit.elementId, + kind: edit.kind, + value: edit.value, + baseVersion: edit.baseVersion, + actor, + }); + if (!result.ok) { + throw new ConvexError({ + code: "artifact_edit_bundle_rejected", + reason: result.reason, + opId: edit.opId, + elementId: edit.elementId, + ...result.reason === "conflict" ? { expected: result.expected, actual: result.actual } : {}, + ...result.reason === "locked" ? { by: result.by } : {}, + ...result.reason === "lease_expired" ? { lockId: result.lockId } : {}, + }); + } + results.push({ opId: edit.opId, elementId: edit.elementId, version: result.version }); + } + + const artifact = await ctx.db.get(a.artifactId); + if (!artifact || String(artifact.roomId) !== String(a.roomId)) { + throw new ConvexError({ code: "artifact_edit_bundle_rejected", reason: "artifact_missing_after_apply" }); + } + return { ok: true as const, artifactVersion: artifact.version, results }; + }, +}); + export const startAgentIntentConflictProof = mutation({ args: { roomId: v.id("rooms"), diff --git a/convex/messages.ts b/convex/messages.ts index f02929e6..868b5f35 100644 --- a/convex/messages.ts +++ b/convex/messages.ts @@ -69,13 +69,14 @@ type SendArgs = { text: string; clientMsgId: string; kind?: "chat" | "agent" | "system"; + jobId?: Id<"agentJobs">; }; async function sendCore(ctx: MutationCtx, a: SendArgs) { await requireActorCanUseChannel(ctx, a.roomId, a.author, a.channel); const existing = await ctx.db.query("messages").withIndex("by_clientMsgId", (q) => q.eq("roomId", a.roomId).eq("clientMsgId", a.clientMsgId)).unique(); if (existing) return existing._id; // idempotent send - const messageId = await ctx.db.insert("messages", { roomId: a.roomId, channel: a.channel, author: a.author, text: a.text, clientMsgId: a.clientMsgId, kind: a.kind ?? "chat", createdAt: Date.now() }); + const messageId = await ctx.db.insert("messages", { roomId: a.roomId, channel: a.channel, author: a.author, text: a.text, clientMsgId: a.clientMsgId, kind: a.kind ?? "chat", jobId: a.jobId, createdAt: Date.now() }); // NodeMem recording (production wiring): a ROOM-VISIBLE chat message (channel "public") is recorded // as an episode so it stays recallable after it scrolls past the awareness window. PRIVATE messages // (channel = a member's ownerId) and system messages are excluded. Gated → a strict no-op unless diff --git a/convex/schema.ts b/convex/schema.ts index 866cd6a2..67990cad 100644 --- a/convex/schema.ts +++ b/convex/schema.ts @@ -357,6 +357,8 @@ export default defineSchema({ text: v.string(), clientMsgId: v.string(), kind: v.union(v.literal("chat"), v.literal("agent"), v.literal("system")), + /** Durable correlation for agent-authored messages. Legacy rows remain valid. */ + jobId: v.optional(v.id("agentJobs")), createdAt: v.number(), /** persistent-text-streaming stream id: while set and text is empty, the body lives in the * streaming component (token-level for the driving tab, sentence-flushed for viewers); on @@ -955,6 +957,7 @@ export default defineSchema({ modelPolicy: v.string(), runtime: v.optional(v.union(v.literal("inline"), v.literal("scheduler"), v.literal("workflow"))), workflowId: v.optional(v.string()), + waitingForJobId: v.optional(v.id("agentJobs")), workId: v.optional(v.string()), activeFrameId: v.optional(v.string()), cursor: v.optional(v.any()), @@ -985,6 +988,7 @@ export default defineSchema({ completedAt: v.optional(v.number()), }) .index("by_room", ["roomId", "updatedAt"]) + .index("by_waiting_for", ["waitingForJobId", "createdAt"]) .index("by_status_nextRunAt", ["status", "nextRunAt"]) .index("by_idempotency", ["idempotencyKey", "createdAt"]), diff --git a/convex/streaming.ts b/convex/streaming.ts index 74bd869a..0fc3e04d 100644 --- a/convex/streaming.ts +++ b/convex/streaming.ts @@ -127,7 +127,7 @@ export const ensurePublicAgentJobStream = internalMutation({ const streamId = String(await streamingComponent.createStream(ctx)); if (existingMessage) { - await ctx.db.patch(existingMessage._id, { streamId, text: "" }); + await ctx.db.patch(existingMessage._id, { streamId, text: "", jobId: a.jobId }); } else { await ctx.db.insert("messages", { roomId: a.roomId, @@ -136,6 +136,7 @@ export const ensurePublicAgentJobStream = internalMutation({ text: "", clientMsgId, kind: "agent", + jobId: a.jobId, createdAt: a.createdAt ?? Date.now(), streamId, }); diff --git a/docs/design/COMPONENT_MAP.md b/docs/design/COMPONENT_MAP.md index d1cadfcb..2aef5a0f 100644 --- a/docs/design/COMPONENT_MAP.md +++ b/docs/design/COMPONENT_MAP.md @@ -71,7 +71,8 @@ Migrated component mappings in this slice: | Binder rail rows/search/upload/review/person groups | Cloud panel and row skin | `src/ui/LeftRail.tsx`, `src/ui/primitives/primitives.css` | Artifact open, proposal open, search filtering, people/agent data rendering. | | Chat composer/messages/run receipts | Cloud message and composer skin | `src/ui/Chat.tsx`, `src/ui/primitives/primitives.css` | Send/edit/retry, mentions, attachments, agent job controls, artifact refs, lane behavior. | | Work surface tabs/grid/notebook/trace entry | Cloud work-surface skin | `src/ui/panels/Artifact.tsx`, `src/ui/primitives/primitives.css` | Tab switching, grid editing, selected cells, trace/notebook/wall/report surfaces, inline rename. | -| Spreadsheet, notebook, proposal, coach, and specialized trace interiors | Cloud interior skin | `src/ui/primitives/primitives.css`, `src/ui/panels/notebook-paper.css`, `src/ui/panels/trace-run.css`, `tests/notebookPaper.test.tsx` | Cell editing, selected cells, render windowing, proposal approval/rejection, notebook ProseMirror sync/blur commit, trace record tabs, flow selection, observability export, run span expansion. | +| Spreadsheet, proposal, coach, and specialized trace interiors | Cloud interior skin | `src/ui/primitives/primitives.css`, `src/ui/panels/trace-run.css` | Cell editing, selected cells, render windowing, proposal approval/rejection, trace record tabs, flow selection, observability export, run span expansion. | +| Desktop notebook paper, intelligence, and artifact-detail interior | `CloudNotebookSurface`, deferred `NotebookIntelligenceTray`, dense `NotebookDigestWorkbench` | `src/ui/panels/Artifact.tsx`, `src/ui/panels/notebook-paper.css`, `src/ui/workArtifacts/WorkArtifactsPanel.tsx`, `src/ui/workArtifacts/NotebookDigestWorkbench.tsx`, `src/ui/workArtifacts/work-artifacts.css` | ProseMirror/legacy editing, blur commits, typed block read model, kernel run/cancel/output persistence, patch review/approval, citations, provenance, hashes, loading/error/empty states. Visual changes may remove cream/serif paper, compact typography and whitespace, flatten rows, and defer intelligence behind a 34px disclosure. | | Command palette, people panel, guided tour, dialogs, form states | Shared overlay/form/state skin | `src/ui/primitives/primitives.css`, `src/ui/RoomShell.tsx` | Keyboard navigation, focus trap, outside click/Escape close, follow behavior, ARIA/test contracts. | | Standalone mobile surface | Cloud mobile token overlay and frame chrome | `src/ui/mobile/mobile.css`, `src/ui/mobile/mobileFrame.css` | Mobile router, sample/live modes, sheets, gestures, composer, note capture, join/consent mechanics. | | Always-On public room modal/forms/errors | Shared public-room/overlay skin | `src/ui/primitives/primitives.css` | Public tabs, subscribe form validation, double-opt-in honesty, modal close/focus behavior. | diff --git a/docs/design/UI_CONTRACT.md b/docs/design/UI_CONTRACT.md index d6471b69..831f98a6 100644 --- a/docs/design/UI_CONTRACT.md +++ b/docs/design/UI_CONTRACT.md @@ -155,7 +155,7 @@ Tokens extracted from the standalone artifact: | Token area | Contract | |---|---| -| Fonts | UI/display: Inter/system stack. Mono: JetBrains Mono. Notebook paper may use Lora/serif. | +| Fonts | UI/display and notebook: Inter/system stack. Mono: JetBrains Mono. The desktop notebook must not introduce a serif paper theme inside the Cloud shell. | | Accent | Terracotta selection/focus: `#D97757`; hover: `#C76648`; warm ink: `#E59579`/`#AD5F45`. | | Secondary signal | Indigo `#5E6AD2`/`#8C92E0` for non-primary status or agent metadata, not as a page-wide theme. | | Semantic colors | Success green only for completed/healthy states. Warning amber for held/review states. Danger red only for errors/failures. | @@ -175,6 +175,38 @@ Layout rules: - Work surfaces should preserve stable dimensions for grids, tab strips, status rails, icon buttons, counters, and cells so state changes do not resize the layout. + +### Desktop notebook interior correction (2026-07-15) + +The dense Cloud shell geometry is authoritative for the notebook. The older +cream paper exception is retired on desktop: the notebook is a dark, edge-to-edge +work surface using the same panel, hairline, type, and status tokens as sheets, +chat, and trace. This correction changes presentation and progressive disclosure, +not notebook data, editing, execution, approval, or provenance behavior. + +- Utility bar: 36px dark row; artifact title left, block/review/save state right. +- Editor: full-height independent scroll region, left aligned, `max-width: 92ch`, + `22px 28px 48px` padding; H1 24px, H2 16px, body 13.5px. +- Intelligence: 34px collapsed-by-default disclosure below the editor; expanded + content is internally scrollable and capped at `min(32vh, 320px)`. +- Intelligence rows: flat hairline-separated rows rather than cards on a gray band. +- Kernel, patch, approval, source, hash, citation, review, and error states remain + visible and keyboard reachable when their disclosure is opened. +- Notebook detail mode owns the center work surface; list-level bundle summaries + do not consume vertical space above an open notebook. +- Desktop notebook backgrounds must not use `#fbf8f1`, `#f7f4ed`, or a white page + canvas. The mobile light-terracotta contract remains unchanged. + +Notebook declutter capability ledger: + +| id | selector/component | user promise | guard | backing behavior | disposition | preserve/reverify assertion | +|---|---|---|---|---|---|---| +| N1 | `.nbk-frame`, `.nbk-paper` | Edit and read the notebook | `PRESERVE_CAPABILITY` | ProseMirror/legacy editor state and blur commits | COMPACT | Editor remains editable, scrollable, and reload-persistent. | +| N2 | `notebook-read-model` | Inspect live typed blocks | `PRESERVE_CAPABILITY` | Convex notebook block read model | DEFER | Collapsed trigger is reachable; opening reveals the same typed blocks. | +| N3 | `notebook-kernel-*` | Choose, run, cancel, and inspect kernels | `PRESERVE_CAPABILITY` | Kernel broker and persisted execution receipts | PRESERVE | Safe/Pyodide controls, output, timeout, cancellation, and persistence remain reachable. | +| N4 | notebook patch/approval controls | Review agent changes before mutation | `PRESERVE_CAPABILITY` | Proposal and approval handlers | PRESERVE | Diff, source requirements, approve/reject, and failure states retain test IDs and behavior. | +| N5 | notebook provenance/citation/hash rows | Audit sources and receipts | `PRESERVE_CAPABILITY` | Trace/source/read-model metadata | PRESERVE | Provenance, hashes, citations, and honest uncertainty remain visible on reveal. | +| N6 | cream paper, hero typography, expanded intelligence band | Notebook-specific visual identity | `ORDINARY_CAPABILITY` | Presentation CSS only | REMOVE/DEFER | Dark Cloud tokens, 24px maximum H1, 34px collapsed tray, no lost capability. | - Mobile routes translate rails into sheets/tabs and keep desktop hover details available through explicit controls. diff --git a/docs/nodeagent/AGENT_READY_API.md b/docs/nodeagent/AGENT_READY_API.md index 512299d3..aeb5da30 100644 --- a/docs/nodeagent/AGENT_READY_API.md +++ b/docs/nodeagent/AGENT_READY_API.md @@ -10,6 +10,7 @@ This file is the model-facing contract: every production tool must expose a non- |---|---:|---|---| | `inspect_workbook` | mixed | `instruction` | `instruction` | | `verify_workbook` | mixed | `instruction`, `operations` | `instruction`, `operations` | +| `execute_workbook_structure_repair` | mixed | `instruction`, `repairId` | `instruction`, `repairId` | | `read_range` | read | none | none | | `search_sheet_context` | read | `query` | `query` | | `list_artifacts` | read | none | none | @@ -109,6 +110,31 @@ This file is the model-facing contract: every production tool must expose a non- } ``` +### execute_workbook_structure_repair + +- Purpose: Execute a complete, agent-visible structural workbook repair contract after inspection. +- When to use: Execute a complete, agent-visible structural workbook repair contract after inspection. +- When not to use: Do not use as a hidden shortcut around room permissions, privacy boundaries, or artifact freshness. +- Mutability: mixed. +- Canonical Zod properties: `artifactId`, `instruction`, `repairId`. +- Canonical required fields: `instruction`, `repairId`. +- Provider required fields: `instruction`, `repairId`. +- Expected errors: missing_required_arg; invalid_arg_type. +- Recovery path: Treat tool failures as inputs: inspect `failureKind` or result reason, add the missing argument or re-read state, and stop rather than inventing data. +- Example call: + +```json +{ + "tool": "execute_workbook_structure_repair", + "args": { + "instruction": "example", + "repairId": [ + "example" + ] + } +} +``` + ### read_range - Purpose: Read the current value + version of specific cells. @@ -390,11 +416,11 @@ This file is the model-facing contract: every production tool must expose a non- ### write_locked_cell -- Purpose: Production write path for a simple scalar cell. +- Purpose: Production write path for a scalar, formula, or style-only cell. - When to use: Use for production spreadsheet writes so lock, CAS, review mode, and receipts stay runtime-managed. - When not to use: Do not use when the target artifact, base version, permission, or evidence requirement is unknown; read or search first. - Mutability: write. -- Canonical Zod properties: `artifactId`, `baseVersion`, `base_version`, `cell`, `cellId`, `cellKey`, `cell_id`, `content`, `currentVersion`, `current_version`, `elementId`, `element_id`, `expectedValue`, `expected_value`, `formula`, `id`, `kind`, `newValue`, `new_value`, `numFmt`, `num_fmt`, `numberFormat`, `number_format`, `reason`, `result`, `target`, `targetCell`, `targetId`, `text`, `value`, `version`. +- Canonical Zod properties: `artifactId`, `baseVersion`, `base_version`, `cell`, `cellId`, `cellKey`, `cell_id`, `content`, `currentVersion`, `current_version`, `elementId`, `element_id`, `expectedValue`, `expected_value`, `fontColor`, `font_color`, `formula`, `id`, `kind`, `newValue`, `new_value`, `numFmt`, `num_fmt`, `numberFormat`, `number_format`, `reason`, `result`, `target`, `targetCell`, `targetId`, `text`, `value`, `version`. - Canonical required fields: none. - Provider required fields: none. - Expected errors: missing_required_arg; invalid_arg_type; cas_conflict; lock_blocked; permission_denied. @@ -414,11 +440,11 @@ This file is the model-facing contract: every production tool must expose a non- ### write_locked_cells -- Purpose: Production batch write path for scalar or formula cells. +- Purpose: Production batch write path for scalar, formula, or style-only cells. - When to use: Use for production spreadsheet writes so lock, CAS, review mode, and receipts stay runtime-managed. - When not to use: Do not use when the target artifact, base version, permission, or evidence requirement is unknown; read or search first. - Mutability: write. -- Canonical Zod properties: `artifactId`, `baseVersions`, `base_version`, `base_versions`, `cell`, `cellIds`, `cells`, `content`, `currentVersion`, `currentVersions`, `elementIds`, `expectedValue`, `formula`, `formulas`, `id`, `ids`, `kind`, `kinds`, `newValue`, `newValues`, `new_value`, `numFmt`, `numFmts`, `numberFormat`, `numberFormats`, `ops`, `reason`, `result`, `results`, `target`, `targetCell`, `targetCells`, `targets`, `text`, `values`, `versions`. +- Canonical Zod properties: `artifactId`, `baseVersions`, `base_version`, `base_versions`, `cell`, `cellIds`, `cells`, `content`, `currentVersion`, `currentVersions`, `elementIds`, `expectedValue`, `fontColor`, `fontColors`, `formula`, `formulas`, `id`, `ids`, `kind`, `kinds`, `newValue`, `newValues`, `new_value`, `numFmt`, `numFmts`, `numberFormat`, `numberFormats`, `ops`, `reason`, `result`, `results`, `target`, `targetCell`, `targetCells`, `targets`, `text`, `values`, `versions`. - Canonical required fields: none. - Provider required fields: none. - Expected errors: missing_required_arg; invalid_arg_type; cas_conflict; lock_blocked; permission_denied. @@ -446,7 +472,7 @@ This file is the model-facing contract: every production tool must expose a non- - When to use: Use for production spreadsheet writes so lock, CAS, review mode, and receipts stay runtime-managed. - When not to use: Do not use when the target artifact, base version, permission, or evidence requirement is unknown; read or search first. - Mutability: write. -- Canonical Zod properties: `artifactId`, `baseVersion`, `base_version`, `cell`, `cellId`, `cellKey`, `cell_id`, `confidence`, `content`, `currentVersion`, `current_version`, `elementId`, `element_id`, `error`, `evidence`, `expectedValue`, `expected_value`, `formula`, `id`, `kind`, `newValue`, `new_value`, `normalizedValue`, `numFmt`, `num_fmt`, `numberFormat`, `number_format`, `reason`, `result`, `status`, `target`, `targetCell`, `targetId`, `text`, `value`, `version`. +- Canonical Zod properties: `artifactId`, `baseVersion`, `base_version`, `cell`, `cellId`, `cellKey`, `cell_id`, `confidence`, `content`, `currentVersion`, `current_version`, `elementId`, `element_id`, `error`, `evidence`, `expectedValue`, `expected_value`, `fontColor`, `font_color`, `formula`, `id`, `kind`, `newValue`, `new_value`, `normalizedValue`, `numFmt`, `num_fmt`, `numberFormat`, `number_format`, `reason`, `result`, `status`, `target`, `targetCell`, `targetId`, `text`, `value`, `version`. - Canonical required fields: `evidence`. - Provider required fields: `evidence`. - Expected errors: missing_required_arg; invalid_arg_type; cas_conflict; lock_blocked; permission_denied; evidence_required. @@ -476,7 +502,7 @@ This file is the model-facing contract: every production tool must expose a non- - When to use: Use for production spreadsheet writes so lock, CAS, review mode, and receipts stay runtime-managed. - When not to use: Do not use when the target artifact, base version, permission, or evidence requirement is unknown; read or search first. - Mutability: write. -- Canonical Zod properties: `artifactId`, `baseVersions`, `base_version`, `base_versions`, `cell`, `cellIds`, `cells`, `confidence`, `confidences`, `content`, `currentVersion`, `currentVersions`, `elementIds`, `error`, `errors`, `evidence`, `evidences`, `expectedValue`, `formula`, `formulas`, `id`, `ids`, `kind`, `kinds`, `newValue`, `newValues`, `new_value`, `normalizedValue`, `normalizedValues`, `ops`, `reason`, `result`, `results`, `status`, `statuses`, `target`, `targetCell`, `targetCells`, `targets`, `text`, `values`, `versions`. +- Canonical Zod properties: `artifactId`, `baseVersions`, `base_version`, `base_versions`, `cell`, `cellIds`, `cells`, `confidence`, `confidences`, `content`, `currentVersion`, `currentVersions`, `elementIds`, `error`, `errors`, `evidence`, `evidences`, `expectedValue`, `fontColor`, `fontColors`, `formula`, `formulas`, `id`, `ids`, `kind`, `kinds`, `newValue`, `newValues`, `new_value`, `normalizedValue`, `normalizedValues`, `ops`, `reason`, `result`, `results`, `status`, `statuses`, `target`, `targetCell`, `targetCells`, `targets`, `text`, `values`, `versions`. - Canonical required fields: none. - Provider required fields: none. - Expected errors: missing_required_arg; invalid_arg_type; cas_conflict; lock_blocked; permission_denied; evidence_required. diff --git a/e2e/deployed-auth-first-user.spec.ts b/e2e/deployed-auth-first-user.spec.ts index f4a6ca80..36ba77fb 100644 --- a/e2e/deployed-auth-first-user.spec.ts +++ b/e2e/deployed-auth-first-user.spec.ts @@ -325,7 +325,7 @@ test.describe("deployed authenticated first-user journey", () => { await expect(page.getByTestId("mobile-header")).toBeVisible({ timeout: 60_000 }); await expect(page.getByTestId("gap-firstjoin")).toBeVisible(); await page.getByRole("button", { name: "Dismiss first-join welcome" }).click(); - await expect(page.getByTestId("mobile-sample-banner")).toContainText("Sample workspace.", { timeout: 90_000 }); + await expect(page.getByTestId("mobile-sample-banner")).toContainText("Live runtime · sample data.", { timeout: 90_000 }); const deckCard = page.locator('.na-rcard[data-kind="deck"]'); await expect(deckCard).toHaveCount(1, { timeout: 60_000 }); diff --git a/e2e/mobile-story-surfaces.spec.ts b/e2e/mobile-story-surfaces.spec.ts index c3d503ce..38f7cc19 100644 --- a/e2e/mobile-story-surfaces.spec.ts +++ b/e2e/mobile-story-surfaces.spec.ts @@ -163,7 +163,7 @@ test.describe("#mobile - terra surface renders (live Convex room)", () => { expect(page.url(), "live mobile proof must still avoid memory mode after room creation").not.toContain("mode=memory"); await expect(app).toHaveCSS("background-color", "rgb(251, 244, 231)"); await expect(app).toContainText(/Startup Banking Diligence War Room/i); - await expect(page.getByTestId("mobile-sample-banner")).toContainText(/Sample (workspace|still loading)/i); + await expect(page.getByTestId("mobile-sample-banner")).toContainText(/Live runtime · sample data/i); await expect(page.locator('[data-testid="ao-room"]')).toHaveCount(0); const firstJoin = page.getByTestId("gap-firstjoin"); await expect(firstJoin).toContainText(/1 person is & 1 agent here/i); diff --git a/e2e/trace-tab.spec.ts b/e2e/trace-tab.spec.ts index dda7fb8c..0ebbbc00 100644 --- a/e2e/trace-tab.spec.ts +++ b/e2e/trace-tab.spec.ts @@ -1,5 +1,11 @@ import { test, expect, enterDemoRoom } from "./fixtures"; +async function openTraceRecords(page: Parameters[0]) { + await page.getByTestId("trace-tab").click(); + await expect(page.getByTestId("trace-view-runs")).toHaveAttribute("aria-selected", "true"); + await page.getByTestId("trace-view-records").click(); +} + /** * Trace work-surface tab — a banker audits provenance after agent work + a QA run. * The tab sits alongside the artifacts; it lists the live agent's source-backed claims and a real @@ -10,7 +16,7 @@ test.describe("trace work-surface tab", () => { await page.setViewportSize({ width: 1440, height: 900 }); await enterDemoRoom(page); - await page.getByTestId("trace-tab").click(); + await openTraceRecords(page); await expect(page.getByTestId("trace-surface")).toBeVisible(); expect(await page.getByTestId("trace-record").count()).toBeGreaterThanOrEqual(2); @@ -30,7 +36,7 @@ test.describe("trace work-surface tab", () => { await page.setViewportSize({ width: 1440, height: 900 }); await enterDemoRoom(page); - await page.getByTestId("trace-tab").click(); + await openTraceRecords(page); await page.getByTestId("trace-record").first().click(); // the live agent record await page.getByTestId("trace-tab-steps").click(); @@ -46,7 +52,7 @@ test.describe("trace work-surface tab", () => { await page.setViewportSize({ width: 1440, height: 900 }); await enterDemoRoom(page); - await page.getByTestId("trace-tab").click(); + await openTraceRecords(page); await page.getByTestId("trace-record").filter({ hasText: "walkthrough" }).first().click(); await page.getByTestId("trace-tab-steps").click(); // Steps are grouped (collapsible) and carry a frame-Δ flicker signal — the QA-automation pipeline. @@ -61,7 +67,7 @@ test.describe("trace work-surface tab", () => { await page.setViewportSize({ width: 1440, height: 900 }); await enterDemoRoom(page); - await page.getByTestId("trace-tab").click(); + await openTraceRecords(page); expect(await page.getByTestId("trace-record").count()).toBeGreaterThanOrEqual(5); // (a) web-source retrieval: the live page is screenshotted with a highlight box on the retrieved value. @@ -79,7 +85,7 @@ test.describe("trace work-surface tab", () => { await page.setViewportSize({ width: 1440, height: 900 }); await enterDemoRoom(page); - await page.getByTestId("trace-tab").click(); + await openTraceRecords(page); await page.getByTestId("trace-record").filter({ hasText: "ledger consolidation" }).first().click(); await expect(page.getByText("Shippable without review? NO", { exact: false }).first()).toBeVisible(); @@ -92,7 +98,7 @@ test.describe("trace work-surface tab", () => { await page.setViewportSize({ width: 1440, height: 900 }); await enterDemoRoom(page); - await page.getByTestId("trace-tab").click(); + await openTraceRecords(page); await page.getByTestId("trace-record").filter({ hasText: "ledger consolidation" }).first().click(); await page.getByTestId("trace-tab-flow").click(); @@ -117,7 +123,7 @@ test.describe("trace work-surface tab", () => { await page.setViewportSize({ width: 1440, height: 900 }); await enterDemoRoom(page); - await page.getByTestId("trace-tab").click(); + await openTraceRecords(page); await page.getByTestId("trace-record").filter({ hasText: "QA" }).first().click(); await page.getByTestId("trace-tab-observability").click(); diff --git a/scripts/openrouter-free-discover.ts b/scripts/openrouter-free-discover.ts index ee5142b4..f0785ea3 100644 --- a/scripts/openrouter-free-discover.ts +++ b/scripts/openrouter-free-discover.ts @@ -33,9 +33,18 @@ if (smoke) { liveProbes.push({ kind: "text", status: "skipped", reason: "missing OPENROUTER_API_KEY" }); } else { const startedAt = Date.now(); - const text = await judge("openrouter/free-auto", "Reply with exactly: OK"); - console.log(`SMOKE openrouter/free-auto -> ${JSON.stringify(text.slice(0, 80))}`); - liveProbes.push({ kind: "text", status: /^\s*OK\s*$/i.test(text) ? "passed" : "failed", requestedModel: "openrouter/free-auto", durationMs: Date.now() - startedAt, preview: text.slice(0, 80) }); + try { + const text = await judge("openrouter/free-auto", "Reply with exactly: OK"); + const passed = /^\s*OK\s*$/i.test(text); + console.log(`SMOKE openrouter/free-auto -> ${JSON.stringify(text.slice(0, 80))}`); + liveProbes.push({ kind: "text", status: passed ? "passed" : "failed", requestedModel: "openrouter/free-auto", durationMs: Date.now() - startedAt, preview: text.slice(0, 80) }); + if (!passed) process.exitCode = 1; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + console.log(`SMOKE openrouter/free-auto -> failed: ${message}`); + liveProbes.push({ kind: "text", status: "failed", requestedModel: "openrouter/free-auto", durationMs: Date.now() - startedAt, error: message }); + process.exitCode = 1; + } } } @@ -46,33 +55,48 @@ if (agentSmoke) { } else { const startedAt = Date.now(); const route = model("openrouter/free-auto"); - const res = await route.next({ - system: "You are a tool-using smoke test. When asked, call the report_answer tool exactly once.", - messages: [{ role: "user", content: "Call report_answer with value OK." }], - tools: [{ - name: "report_answer", - description: "Report a short answer.", - schema: z.object({ value: z.string() }), - execute: async () => ({ ok: true }), - }], - }); - const first = res.toolCalls[0]; - const passed = first?.tool === "report_answer" && String(first.args.value) === "OK"; - console.log(`AGENT_SMOKE ${route.name} -> ${first?.tool ?? "no_tool"} ${JSON.stringify(first?.args ?? {})}`); - liveProbes.push({ - kind: "agent_tool", - status: passed ? "passed" : "failed", - requestedModel: "openrouter/free-auto", - resolvedModel: route.name, - durationMs: Date.now() - startedAt, - inputTokens: res.usage?.inputTokens ?? 0, - outputTokens: res.usage?.outputTokens ?? 0, - providerCostUsd: 0, - expectedTool: "report_answer", - actualTool: first?.tool ?? null, - actualArgs: first?.args ?? {}, - }); - if (!passed) process.exitCode = 1; + try { + const res = await route.next({ + system: "You are a tool-using smoke test. When asked, call the report_answer tool exactly once.", + messages: [{ role: "user", content: "Call report_answer with value OK." }], + tools: [{ + name: "report_answer", + description: "Report a short answer.", + schema: z.object({ value: z.string() }), + execute: async () => ({ ok: true }), + }], + }); + const first = res.toolCalls[0]; + const passed = first?.tool === "report_answer" && String(first.args.value) === "OK"; + console.log(`AGENT_SMOKE ${route.name} -> ${first?.tool ?? "no_tool"} ${JSON.stringify(first?.args ?? {})}`); + liveProbes.push({ + kind: "agent_tool", + status: passed ? "passed" : "failed", + requestedModel: "openrouter/free-auto", + resolvedModel: route.name, + durationMs: Date.now() - startedAt, + inputTokens: res.usage?.inputTokens ?? 0, + outputTokens: res.usage?.outputTokens ?? 0, + providerCostUsd: 0, + expectedTool: "report_answer", + actualTool: first?.tool ?? null, + actualArgs: first?.args ?? {}, + }); + if (!passed) process.exitCode = 1; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + console.log(`AGENT_SMOKE ${route.name} -> failed: ${message}`); + liveProbes.push({ + kind: "agent_tool", + status: "failed", + requestedModel: "openrouter/free-auto", + resolvedModel: route.name, + durationMs: Date.now() - startedAt, + providerCostUsd: 0, + error: message, + }); + process.exitCode = 1; + } } } diff --git a/scripts/spreadsheetbench-accept-official-v2.ts b/scripts/spreadsheetbench-accept-official-v2.ts index f846ebf8..a2e59625 100644 --- a/scripts/spreadsheetbench-accept-official-v2.ts +++ b/scripts/spreadsheetbench-accept-official-v2.ts @@ -1,6 +1,7 @@ import { createHash } from "node:crypto"; import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { dirname, join, relative, resolve } from "node:path"; +import { isSpreadsheetBenchOfficialV2AcceptedRefreshStatus } from "../src/eval/spreadsheetBenchOfficialV2Acceptance"; type ProjectionReceipt = { schema?: number; @@ -89,7 +90,7 @@ const projectedOutputs = new Set((projection.cases ?? []).map((item) => item.out const refreshRecords = refresh.records ?? []; if (projectedOutputs.size !== 321 || refreshRecords.length !== 321) throw new Error("projection/refresh output sets must each contain 321 files"); for (const record of refreshRecords) { - if (!record.path || record.status !== "refreshed" || !/^[a-f0-9]{64}$/i.test(record.afterSha256 ?? "")) { + if (!record.path || !isSpreadsheetBenchOfficialV2AcceptedRefreshStatus(record.status) || !/^[a-f0-9]{64}$/i.test(record.afterSha256 ?? "")) { throw new Error(`invalid refresh record: ${String(record.path)}`); } const absolute = resolve(record.path); diff --git a/scripts/spreadsheetbench-official-v2-project.ts b/scripts/spreadsheetbench-official-v2-project.ts index 4df29812..c69765eb 100644 --- a/scripts/spreadsheetbench-official-v2-project.ts +++ b/scripts/spreadsheetbench-official-v2-project.ts @@ -115,6 +115,11 @@ const NODEAGENT_MUTATION_TOOLS = new Set([ "write_locked_cell", "write_locked_cells", "execute_verified_workbook_plan", + "execute_workbook_structure_repair", +]); +const NODEAGENT_COMPOSITE_MUTATION_TOOLS = new Set([ + "execute_verified_workbook_plan", + "execute_workbook_structure_repair", ]); const args = process.argv.slice(2); @@ -800,11 +805,24 @@ function validateNodeAgentExecutionConsistency( committedMutationCount += 1; committedTargetOccurrences += targets.length; } + const targetIds: string[] = []; for (const value of targets) { const target = asObject(value); if (!target || !nonemptyString(target.refId) || !nonemptyString(target.kind)) { return "NodeAgent mutation target is invalid"; } + targetIds.push(target.refId as string); + } + if (event.tool === "execute_workbook_structure_repair") { + const structuralTargets = asObject(event.result)?.targets; + if ( + !Array.isArray(structuralTargets) || + structuralTargets.length === 0 || + structuralTargets.some((target) => !nonemptyString(target)) || + !isDeepStrictEqual(targetIds, structuralTargets) + ) { + return "NodeAgent structural mutation targets do not bind its repair result"; + } } } const observedCommittedMutation = committedMutationCount > 0; @@ -970,8 +988,24 @@ function traceMutationStatus( ): "proposed" | "committed" | "skipped" | "conflict" | "pending_approval" { const record = asObject(result); if (record?.pendingApproval === true) return "pending_approval"; + const resultItems = Array.isArray(record?.results) + ? record.results.map(asObject).filter((item): item is JsonObject => Boolean(item)) + : []; + const hasCommittedResult = resultItems.some( + (item) => item.ok === true && item.skipped !== true, + ); + if ( + hasCommittedResult || + (typeof record?.changedTargetCount === "number" && record.changedTargetCount > 0) + ) { + return "committed"; + } + if (record?.alreadySatisfied === true) return "skipped"; if (record?.conflict === true) return "conflict"; if (record?.skipped === true) return "skipped"; + if (resultItems.length > 0 && resultItems.every((item) => item.skipped === true)) { + return "skipped"; + } if (record?.ok === true) return "committed"; return "conflict"; } @@ -988,7 +1022,7 @@ function deriveNodeAgentStages( const indexed = frameEvents.map((event, eventIndex) => ({ event, eventIndex })); const inspect = indexed.filter(({ event }) => event.tool === "inspect_workbook"); const composite = indexed.filter( - ({ event }) => event.tool === "execute_verified_workbook_plan", + ({ event }) => NODEAGENT_COMPOSITE_MUTATION_TOOLS.has(String(event.tool)), ); const verifications = indexed.filter( ({ event }) => event.tool === "verify_workbook", @@ -1151,7 +1185,7 @@ function nodeAgentEventVerificationStatus( phase: "preflight" | "verify", ): "passed" | "needs_repair" | "missing" { if (!event) return "missing"; - return event.tool === "execute_verified_workbook_plan" + return NODEAGENT_COMPOSITE_MUTATION_TOOLS.has(String(event.tool)) ? nodeAgentCompositePhaseStatus(event.result, phase) : nodeAgentVerificationStatus(event.result); } diff --git a/scripts/spreadsheetbench-refresh-excel.py b/scripts/spreadsheetbench-refresh-excel.py index 221f511d..51a0990f 100644 --- a/scripts/spreadsheetbench-refresh-excel.py +++ b/scripts/spreadsheetbench-refresh-excel.py @@ -1,4 +1,4 @@ -"""Refresh cached workbook values with local Excel and emit a hash-bound receipt.""" +"""Finalize workbook formula caches with fail-closed Excel evidence.""" from __future__ import annotations @@ -10,8 +10,6 @@ import os import posixpath import re -import shutil -import subprocess import tempfile import time import zipfile @@ -42,12 +40,30 @@ def sha256_file(path: Path) -> str: rf'<(?P{XML_PREFIX}v)\b(?P[^>]*?)(?:(?P/>)|>(?P[\s\S]*?))', re.IGNORECASE, ) +INLINE_STRING_RE = re.compile( + rf'<(?P{XML_PREFIX}is)\b(?P[^>]*?)>(?P[\s\S]*?)', + re.IGNORECASE, +) CELL_REF_RE = re.compile(r'\br\s*=\s*["\']([^"\']+)["\']', re.IGNORECASE) CELL_TYPE_RE = re.compile(r'\s+t\s*=\s*(["\'])[\s\S]*?\1', re.IGNORECASE) ADDRESS_RE = re.compile(r'^([A-Z]{1,3})([1-9][0-9]*)$') RANGE_RE = re.compile(r'^\$?([A-Z]{1,3})\$?([1-9][0-9]*)(?::\$?([A-Z]{1,3})\$?([1-9][0-9]*))?$') EXCEL_ESCAPE_RE = re.compile(r'_[xX][0-9A-Fa-f]{4}_') MAX_FORMULA_CACHE_TARGETS_PER_SHEET = 1_000_000 +SUPPORTED_FORMULA_TOPOLOGIES = ("normal", "shared", "array", "dataTable") +FINALIZATION_STATUSES = ( + "completed", + "completed_stable_pending", + "not_required", + "preserved_pending", + "preserved_unsupported", + "preserved_error", +) +PRESERVED_STATUSES = { + "preserved_pending", + "preserved_unsupported", + "preserved_error", +} EXCEL_ERROR_CODES = { 2000: "#NULL!", 2007: "#DIV/0!", @@ -75,13 +91,33 @@ class FormulaCache: value_xml: str -def refresh_with_excel(excel, path: Path, record: dict[str, object]) -> None: - formula_cells = workbook_formula_cells(path) - formula_count = sum(len(addresses) for _, addresses in formula_cells.values()) - record["formulaCellCount"] = formula_count - if formula_count == 0: - record["cacheWriteMode"] = "no_formula_caches" - return +@dataclass(frozen=True) +class FormulaTopologyInspection: + formula_cells: dict[str, tuple[str, list[str]]] + formula_cell_count: int + detail: dict[str, object] + + @property + def safe(self) -> bool: + return bool(self.detail["safe"]) + + +class ExcelCalculationPending(RuntimeError): + pass + + +class FormulaCachePatchError(RuntimeError): + pass + + +def refresh_with_excel( + excel, + path: Path, + record: dict[str, object], + formula_cells: dict[str, tuple[str, list[str]]], + *, + allow_stable_pending: bool = False, +) -> None: workbook = None open_options = { "UpdateLinks": 0, @@ -91,86 +127,166 @@ def refresh_with_excel(excel, path: Path, record: dict[str, object]) -> None: } try: try: - workbook = excel.Workbooks.Open(str(path), **open_options) - record["openMode"] = "normal" + workbook = excel.Workbooks.Open(str(path.resolve()), **open_options) + record["openMode"] = "normal_read_only" except Exception as normal_open_error: record["normalOpenError"] = str(normal_open_error) - workbook = excel.Workbooks.Open(str(path), CorruptLoad=1, **open_options) - record["openMode"] = "repair" + workbook = excel.Workbooks.Open(str(path.resolve()), CorruptLoad=1, **open_options) + record["openMode"] = "repair_read_only" excel.CalculateFullRebuild() - values = collect_stable_formula_values(excel, workbook, formula_cells, record) + values = collect_formula_values_with_state_gate( + excel, + workbook, + formula_cells, + record, + allow_stable_pending=allow_stable_pending, + ) finally: if workbook is not None: workbook.Close(SaveChanges=False) - patch_formula_caches( - path, - formula_cells, - values, - expected_sha256=str(record["beforeSha256"]), - ) + try: + topology_preservation = patch_formula_caches( + path, + formula_cells, + values, + expected_sha256=str(record.get("preRefreshSha256", record["beforeSha256"])), + ) + except Exception as exc: + raise FormulaCachePatchError(str(exc)) from exc record["cacheWriteMode"] = "original_package_formula_cache_patch" + record["formulaTopologyPreservation"] = topology_preservation -def refresh_with_libreoffice(path: Path) -> str: - soffice = shutil.which("soffice") or r"C:\Program Files\LibreOffice\program\soffice.exe" - if not Path(soffice).exists(): - raise RuntimeError("LibreOffice soffice executable is unavailable") - with tempfile.TemporaryDirectory(prefix="ssb-v2-lo-") as temp_dir: - temp_root = Path(temp_dir) - output_dir = temp_root / "output" - profile_dir = temp_root / "profile" - output_dir.mkdir() - command = [ - soffice, - "--headless", - "--nologo", - "--nodefault", - "--nolockcheck", - "--norestore", - f"-env:UserInstallation={profile_dir.as_uri()}", - "--convert-to", - "xlsx", - "--outdir", - str(output_dir), - str(path), - ] - result = subprocess.run(command, capture_output=True, text=True, timeout=180) - converted = output_dir / path.name - if result.returncode != 0 or not converted.exists(): - detail = (result.stderr or result.stdout or "no converted workbook").strip() - raise RuntimeError(f"LibreOffice conversion failed ({result.returncode}): {detail}") - shutil.copy2(converted, path) - return (result.stdout or result.stderr).strip() - - -def workbook_formula_cells(path: Path) -> dict[str, tuple[str, list[str]]]: +def inspect_formula_topology(path: Path) -> FormulaTopologyInspection: + topology_counts = { + "normal": 0, + "shared": 0, + "array": 0, + "dataTable": 0, + "unknown": 0, + } + unsupported: list[dict[str, object]] = [] with zipfile.ZipFile(path, "r") as package: sheet_parts = workbook_sheet_parts(package) result: dict[str, tuple[str, list[str]]] = {} for sheet_name, part_path in sheet_parts.items(): xml = package.read(part_path).decode("utf-8") worksheet = ElementTree.fromstring(xml) - addresses: set[str] = set() + cells_by_address: dict[str, ElementTree.Element] = {} for cell in worksheet.iter(): if local_name(cell.tag) != "c": continue address = normalize_address(cell.attrib.get("r", "")) + if ADDRESS_RE.fullmatch(address): + cells_by_address[address] = cell + + def is_cache_result_cell(address: str) -> bool: + cell = cells_by_address.get(address) + if cell is None: + return False + # Inline/shared strings are persisted workbook content, not formula + # result caches. A number of financial-model data tables use them + # for display labels even inside the declared data-table rectangle. + if cell.attrib.get("t") in {"inlineStr", "s"}: + return False + return not any(local_name(child.tag) == "is" for child in cell) + formula_addresses: set[str] = set() + cache_targets: set[str] = set() + for cell in worksheet.iter(): + if local_name(cell.tag) != "c": + continue formula = next((child for child in cell if local_name(child.tag) == "f"), None) - if not address or formula is None: + if formula is None: continue - addresses.add(address) - formula_type = formula.attrib.get("t") - formula_ref = formula.attrib.get("ref") - if formula_ref and formula_type in {"array", "dataTable"}: - addresses.update(expand_range_addresses(formula_ref)) - if addresses: - if len(addresses) > MAX_FORMULA_CACHE_TARGETS_PER_SHEET: + address = normalize_address(cell.attrib.get("r", "")) + if not ADDRESS_RE.fullmatch(address): + raise RuntimeError( + f"worksheet {sheet_name} contains a formula cell without a valid address" + ) + if address in formula_addresses: + raise RuntimeError( + f"worksheet {sheet_name} contains duplicate formula cell {address}" + ) + formula_addresses.add(address) + cache_targets.add(address) + declared_type = formula.attrib.get("t") + normalized_type = (declared_type or "normal").strip().casefold() + topology_markers = sorted( + key for key in (local_name(name) for name in formula.attrib) if key != "t" + ) + if normalized_type == "normal" and any( + marker in {"ref", "si", "dt2D", "dtr", "r1", "r2"} + for marker in topology_markers + ): + topology_type = "unknown" + elif normalized_type == "normal": + topology_type = "normal" + elif normalized_type == "shared": + topology_type = "shared" + elif normalized_type == "array": + topology_type = "array" + elif normalized_type == "datatable": + topology_type = "dataTable" + else: + topology_type = "unknown" + topology_counts[topology_type] += 1 + if topology_type in {"shared", "array"} and formula.attrib.get("ref"): + cache_targets.update( + address + for address in expand_range_addresses(formula.attrib["ref"]) + if is_cache_result_cell(address) + ) + elif topology_type == "dataTable" and formula.attrib.get("ref"): + # A two-input data table's top row and left column are inputs/labels, + # not calculated cache cells. Rewriting those cells can leave an + # inline-string payload under t="e" and makes Excel reject the file. + # One-input table shapes are less explicit, so only patch their + # formula anchor until a safe result-cell contract is available. + if formula.attrib.get("dt2D") in {"1", "true", "True"}: + range_addresses = expand_range_addresses(formula.attrib["ref"]) + positions = [parse_address(address) for address in range_addresses] + min_row = min(row for row, _ in positions) + min_col = min(col for _, col in positions) + cache_targets.update( + address + for address in range_addresses + if is_cache_result_cell(address) + and parse_address(address)[0] > min_row + and parse_address(address)[1] > min_col + ) + if topology_type not in SUPPORTED_FORMULA_TOPOLOGIES: + unsupported.append({ + "sheet": sheet_name, + "cell": address, + "type": topology_type, + **({"declaredType": declared_type} if declared_type else {}), + **({"topologyMarkers": topology_markers} if topology_markers else {}), + **({"reference": formula.attrib["ref"]} if formula.attrib.get("ref") else {}), + }) + if cache_targets: + if len(cache_targets) > MAX_FORMULA_CACHE_TARGETS_PER_SHEET: raise RuntimeError( - f"worksheet {sheet_name} declares {len(addresses)} formula cache targets; " + f"worksheet {sheet_name} declares {len(cache_targets)} formula cache targets; " f"limit is {MAX_FORMULA_CACHE_TARGETS_PER_SHEET}" ) - result[sheet_name] = (part_path, sorted(addresses, key=parse_address)) - return result + result[sheet_name] = (part_path, sorted(cache_targets, key=parse_address)) + formula_cell_count = sum(topology_counts.values()) + cache_target_cell_count = sum(len(addresses) for _, addresses in result.values()) + return FormulaTopologyInspection( + formula_cells=result, + formula_cell_count=formula_cell_count, + detail={ + "safe": not unsupported, + "supportedTypes": list(SUPPORTED_FORMULA_TOPOLOGIES), + "counts": topology_counts, + "cacheTargetCellCount": cache_target_cell_count, + "unsupported": unsupported, + }, + ) + + +def workbook_formula_cells(path: Path) -> dict[str, tuple[str, list[str]]]: + return inspect_formula_topology(path).formula_cells def workbook_sheet_parts(package: zipfile.ZipFile) -> dict[str, str]: @@ -251,38 +367,104 @@ def collect_formula_values( return values -def collect_stable_formula_values( +def collect_formula_values_with_state_gate( excel, workbook, formula_cells: dict[str, tuple[str, list[str]]], record: dict[str, object], + *, + allow_stable_pending: bool = False, + required_identical_reads: int = 3, + sample_interval_seconds: float = 0.25, timeout_seconds: float = 10.0, ) -> dict[str, dict[str, Any]]: - state = int(excel.CalculationState) - record["calculationStateAfterFullRebuild"] = state - values = collect_formula_values(workbook, formula_cells) - if state == 0: - record["calculationStabilityCheck"] = "excel_done" - return values + state_before = int(excel.CalculationState) + record["calculationStateBeforeCacheRead"] = state_before + if state_before == 0: + values = collect_formula_values(workbook, formula_cells) + state_after = int(excel.CalculationState) + elif allow_stable_pending: + values, state_after = collect_stable_pending_formula_values( + excel, + workbook, + formula_cells, + record, + required_identical_reads=required_identical_reads, + sample_interval_seconds=sample_interval_seconds, + timeout_seconds=timeout_seconds, + ) + else: + values = None + state_after = int(excel.CalculationState) + record["calculationStateAfterCacheRead"] = state_after + record["calculationStates"] = { + "beforeCacheRead": state_before, + "afterCacheRead": state_after, + } + if state_before != 0 or state_after != 0: + if allow_stable_pending and values is not None and record.get("calculationStability", {}).get("passed") is True: + record["calculationStateGate"] = "accepted_stable_pending" + return values + record["calculationStateGate"] = "preserved_pending" + raise ExcelCalculationPending( + "Excel calculation state must be 0 before and after the cache read " + f"(before={state_before}, after={state_after})" + ) + record["calculationStateGate"] = "passed" + if values is None: + raise RuntimeError("Excel formula cache values were not read") + return values + +def collect_stable_pending_formula_values( + excel, + workbook, + formula_cells: dict[str, tuple[str, list[str]]], + record: dict[str, object], + *, + required_identical_reads: int, + sample_interval_seconds: float, + timeout_seconds: float, +) -> tuple[dict[str, dict[str, Any]] | None, int]: + if required_identical_reads < 2: + raise ValueError("stable-pending finalization requires at least two identical cache reads") deadline = time.monotonic() + timeout_seconds - stable_reads = 0 + values = collect_formula_values(workbook, formula_cells) + identical_reads = 1 + observed_reads = 1 while time.monotonic() < deadline: - time.sleep(0.25) + time.sleep(sample_interval_seconds) current = collect_formula_values(workbook, formula_cells) + observed_reads += 1 if formula_value_maps_equal(values, current): - stable_reads += 1 - if stable_reads >= 2: - record["calculationStabilityCheck"] = "stable_while_excel_pending" - record["calculationStateAfterStabilityCheck"] = int(excel.CalculationState) - return current + identical_reads += 1 + values = current + if identical_reads >= required_identical_reads: + state_after = int(excel.CalculationState) + record["calculationStability"] = { + "mode": "stable_pending_opt_in", + "passed": True, + "requiredIdenticalReads": required_identical_reads, + "observedIdenticalReads": identical_reads, + "observedReads": observed_reads, + "sampleIntervalMs": round(sample_interval_seconds * 1000), + "timeoutMs": round(timeout_seconds * 1000), + } + return values, state_after else: - stable_reads = 0 - values = current - raise RuntimeError( - f"Excel calculation remained unstable for {timeout_seconds:.1f}s " - f"(state={int(excel.CalculationState)})" - ) + identical_reads = 1 + values = current + state_after = int(excel.CalculationState) + record["calculationStability"] = { + "mode": "stable_pending_opt_in", + "passed": False, + "requiredIdenticalReads": required_identical_reads, + "observedIdenticalReads": identical_reads, + "observedReads": observed_reads, + "sampleIntervalMs": round(sample_interval_seconds * 1000), + "timeoutMs": round(timeout_seconds * 1000), + } + return None, state_after def formula_value_maps_equal(left: dict[str, dict[str, Any]], right: dict[str, dict[str, Any]]) -> bool: @@ -413,7 +595,7 @@ def patch_formula_caches( values: dict[str, dict[str, Any]], *, expected_sha256: str, -) -> None: +) -> dict[str, object]: part_values = { part_path: values[sheet_name] for sheet_name, (part_path, _) in formula_cells.items() @@ -425,9 +607,12 @@ def patch_formula_caches( ) os.close(descriptor) replacement = Path(replacement_name) + topology_before: dict[str, object] | None = None + topology_preservation: dict[str, object] | None = None try: if sha256_file(path) != expected_sha256: raise RuntimeError(f"workbook changed before cache patching: {path}") + topology_before = formula_topology_fingerprint(path, part_values) with zipfile.ZipFile(path, "r") as source, zipfile.ZipFile(replacement, "w") as output: source_names = source.namelist() output.comment = source.comment @@ -452,12 +637,207 @@ def patch_formula_caches( bad_entry = candidate.testzip() if bad_entry: raise RuntimeError(f"cache-patched workbook has an invalid ZIP entry: {bad_entry}") + topology_after = formula_topology_fingerprint(replacement, part_values) + if topology_before != topology_after: + raise RuntimeError("cache patch changed workbook formula topology") + topology_preservation = { + "matched": True, + "beforeSha256": topology_before["sha256"], + "afterSha256": topology_after["sha256"], + "formulaElementCount": topology_before["formulaElementCount"], + } if sha256_file(path) != expected_sha256: raise RuntimeError(f"workbook changed concurrently during cache patching: {path}") os.replace(replacement, path) finally: if replacement.exists(): replacement.unlink() + if topology_preservation is None: + raise RuntimeError("cache patch omitted formula-topology preservation evidence") + return topology_preservation + + +def repair_inline_string_cache_conflicts( + path: Path, + *, + expected_sha256: str, +) -> dict[str, object]: + """Remove only impossible cache payloads left beside inline-string content.""" + descriptor, replacement_name = tempfile.mkstemp( + prefix=f".{path.name}.", + suffix=".inline-cache-repaired", + dir=path.parent, + ) + os.close(descriptor) + replacement = Path(replacement_name) + repaired_cells: list[dict[str, str]] = [] + duplicate_value_cells: list[dict[str, str]] = [] + try: + if sha256_file(path) != expected_sha256: + raise RuntimeError(f"workbook changed before inline-string compatibility repair: {path}") + topology_before = formula_topology_fingerprint(path) + with zipfile.ZipFile(path, "r") as source, zipfile.ZipFile(replacement, "w") as output: + source_names = source.namelist() + worksheet_parts = set(workbook_sheet_parts(source).values()) + output.comment = source.comment + for info in source.infolist(): + data = source.read(info.filename) + if info.filename in worksheet_parts: + xml = data.decode("utf-8") + deduplicated_xml, duplicate_addresses, ambiguous_addresses = repair_worksheet_duplicate_value_caches(xml) + if ambiguous_addresses: + raise RuntimeError( + f"worksheet {info.filename} has {len(ambiguous_addresses)} conflicting duplicate value cache(s)" + ) + patched_xml, addresses = repair_worksheet_inline_string_cache_conflicts(deduplicated_xml) + if duplicate_addresses: + duplicate_value_cells.extend( + {"part": info.filename, "cell": address} + for address in sorted(duplicate_addresses, key=parse_address) + ) + if addresses: + repaired_cells.extend( + {"part": info.filename, "cell": address} + for address in sorted(addresses, key=parse_address) + ) + if duplicate_addresses or addresses: + ElementTree.fromstring(patched_xml) + data = patched_xml.encode("utf-8") + output.writestr(info, data) + if not repaired_cells and not duplicate_value_cells: + return { + "changed": False, + "repairedCellCount": 0, + "cells": [], + "duplicateValueCacheCellCount": 0, + "duplicateValueCacheCells": [], + } + with zipfile.ZipFile(replacement, "r") as candidate: + if candidate.namelist() != source_names: + raise RuntimeError("inline-string compatibility repair changed workbook package entries") + bad_entry = candidate.testzip() + if bad_entry: + raise RuntimeError(f"inline-string compatibility repair produced an invalid ZIP entry: {bad_entry}") + topology_after = formula_topology_fingerprint(replacement) + if topology_before != topology_after: + raise RuntimeError("inline-string compatibility repair changed workbook formula topology") + if sha256_file(path) != expected_sha256: + raise RuntimeError(f"workbook changed concurrently during inline-string compatibility repair: {path}") + os.replace(replacement, path) + return { + "changed": True, + "repairedCellCount": len(repaired_cells), + "cells": repaired_cells, + "duplicateValueCacheCellCount": len(duplicate_value_cells), + "duplicateValueCacheCells": duplicate_value_cells, + "formulaTopologyPreservation": { + "matched": True, + "beforeSha256": topology_before["sha256"], + "afterSha256": topology_after["sha256"], + "formulaElementCount": topology_before["formulaElementCount"], + }, + } + finally: + if replacement.exists(): + replacement.unlink() + + +def repair_worksheet_inline_string_cache_conflicts(xml: str) -> tuple[str, set[str]]: + repaired_addresses: set[str] = set() + + def replace_cell(match: re.Match[str]) -> str: + attrs = match.group("attrs") + body = match.group("body") or "" + if not INLINE_STRING_RE.search(body) or not VALUE_RE.search(body): + return match.group(0) + cell_ref = CELL_REF_RE.search(attrs) + address = normalize_address(cell_ref.group(1)) if cell_ref else "" + if not ADDRESS_RE.fullmatch(address): + return match.group(0) + next_attrs = CELL_TYPE_RE.sub("", attrs) + next_body = VALUE_RE.sub("", body) + repaired_addresses.add(address) + return f'<{match.group("tag")}{next_attrs} t="inlineStr">{next_body}' + + return CELL_RE.sub(replace_cell, xml), repaired_addresses + + +def repair_worksheet_duplicate_value_caches(xml: str) -> tuple[str, set[str], set[str]]: + repaired_addresses: set[str] = set() + ambiguous_addresses: set[str] = set() + + def replace_cell(match: re.Match[str]) -> str: + attrs = match.group("attrs") + body = match.group("body") or "" + values = list(VALUE_RE.finditer(body)) + if len(values) <= 1: + return match.group(0) + cell_ref = CELL_REF_RE.search(attrs) + address = normalize_address(cell_ref.group(1)) if cell_ref else "" + if not ADDRESS_RE.fullmatch(address): + return match.group(0) + signatures = { + (value.group("self") or "", value.group("body") or "") + for value in values + } + if len(signatures) != 1: + ambiguous_addresses.add(address) + return match.group(0) + preferred_value = next( + ( + value.group(0) + for value in values + if re.search(r"\bxml:space\s*=\s*['\"]preserve['\"]", value.group("attrs") or "", re.IGNORECASE) + ), + values[0].group(0), + ) + kept = False + + def keep_one(value_match: re.Match[str]) -> str: + nonlocal kept + if kept: + return "" + kept = True + return preferred_value + + repaired_addresses.add(address) + next_body = VALUE_RE.sub(keep_one, body) + return f'<{match.group("tag")}{attrs}>{next_body}' + + return CELL_RE.sub(replace_cell, xml), repaired_addresses, ambiguous_addresses + + +def formula_topology_fingerprint( + path: Path, + part_paths: object | None = None, +) -> dict[str, object]: + digest = hashlib.sha256() + formula_element_count = 0 + with zipfile.ZipFile(path, "r") as package: + selected_parts = ( + sorted(set(str(part_path) for part_path in part_paths)) + if part_paths is not None + else sorted(set(workbook_sheet_parts(package).values())) + ) + for part_path in selected_parts: + xml = package.read(part_path).decode("utf-8") + for cell_match in CELL_RE.finditer(xml): + attrs = cell_match.group("attrs") + body = cell_match.group("body") or "" + cell_ref = CELL_REF_RE.search(attrs) + address = normalize_address(cell_ref.group(1)) if cell_ref else "" + for formula_match in FORMULA_RE.finditer(body): + digest.update(part_path.encode("utf-8")) + digest.update(b"\0") + digest.update(address.encode("ascii")) + digest.update(b"\0") + digest.update(formula_match.group(0).encode("utf-8")) + digest.update(b"\0") + formula_element_count += 1 + return { + "sha256": digest.hexdigest(), + "formulaElementCount": formula_element_count, + } def patch_worksheet_formula_caches(xml: str, values: dict[str, Any]) -> tuple[str, set[str]]: @@ -478,15 +858,21 @@ def replace_cell(match: re.Match[str]) -> str: cell_tag = match.group("tag") value_tag = f'{cell_tag.rsplit(":", 1)[0]}:v' if ":" in cell_tag else "v" + value_replaced = False + def replace_value(value_match: re.Match[str]) -> str: + nonlocal value_replaced + if value_replaced: + return "" + value_replaced = True return ( f'<{value_match.group("tag")}{value_match.group("attrs")}>' f'{cache.value_xml}' ) cached_value = f"<{value_tag}>{cache.value_xml}" - next_body = VALUE_RE.sub(replace_value, body, count=1) - if next_body == body: + next_body = VALUE_RE.sub(replace_value, body) + if not value_replaced: formula_match = FORMULA_RE.search(body) if formula_match: next_body = f"{body[:formula_match.end()]}{cached_value}{body[formula_match.end():]}" @@ -525,72 +911,6 @@ def encode_spreadsheetml_string(value: str) -> str: return html.escape(encoded, quote=False) -def refresh_with_libreoffice_cache_patch(path: Path, record: dict[str, object]) -> None: - formula_cells = workbook_formula_cells(path) - formula_count = sum(len(addresses) for _, addresses in formula_cells.values()) - record["formulaCellCount"] = formula_count - if formula_count == 0: - record["cacheWriteMode"] = "no_formula_caches" - return - with tempfile.TemporaryDirectory(prefix="ssb-v2-cache-fallback-") as temp_dir: - temporary = Path(temp_dir) / path.name - shutil.copy2(path, temporary) - record["libreOfficeOutput"] = refresh_with_libreoffice(temporary) - values = read_formula_caches(temporary, formula_cells) - patch_formula_caches( - path, - formula_cells, - values, - expected_sha256=str(record["beforeSha256"]), - ) - record["cacheWriteMode"] = "original_package_formula_cache_patch" - - -def read_formula_caches( - path: Path, - source_formula_cells: dict[str, tuple[str, list[str]]], -) -> dict[str, dict[str, FormulaCache]]: - with zipfile.ZipFile(path, "r") as package: - refreshed_parts = workbook_sheet_parts(package) - values: dict[str, dict[str, FormulaCache]] = {} - for sheet_name, (_, addresses) in source_formula_cells.items(): - refreshed_path = refreshed_parts.get(sheet_name) - if not refreshed_path: - raise RuntimeError(f"refreshed workbook is missing worksheet {sheet_name}") - xml = package.read(refreshed_path).decode("utf-8") - by_address: dict[str, FormulaCache] = {} - requested = set(addresses) - for match in CELL_RE.finditer(xml): - cell_ref = CELL_REF_RE.search(match.group("attrs")) - address = cell_ref.group(1).replace("$", "").upper() if cell_ref else "" - if address not in requested: - continue - body = match.group("body") or "" - value_match = VALUE_RE.search(body) - raw = value_match.group("body") or "" if value_match else "" - cell_type = next( - (item.group(2) for item in re.finditer(r'\bt\s*=\s*(["\'])([^"\']+)\1', match.group("attrs"), re.IGNORECASE)), - None, - ) - if cell_type == "inlineStr": - text = "".join( - html.unescape(text_match.group(1)) - for text_match in re.finditer( - rf'<{XML_PREFIX}t\b[^>]*>([\s\S]*?)', - body, - re.IGNORECASE, - ) - ) - by_address[address] = FormulaCache("str", encode_spreadsheetml_string(text)) - else: - by_address[address] = FormulaCache(cell_type, raw) - missing = requested.difference(by_address) - if missing: - raise RuntimeError(f"refreshed workbook omitted {len(missing)} formula cache cell(s) on {sheet_name}") - values[sheet_name] = by_address - return values - - def create_excel_application(): import win32com.client # Imported lazily so OOXML helpers remain cross-platform testable. @@ -625,29 +945,304 @@ def select_workbooks(directory: str | None, file: str | None) -> tuple[Path, lis return root, sorted(root.rglob("*_output.xlsx")) +def receipt_workbook_path(path: Path) -> str: + try: + return path.relative_to(Path.cwd()).as_posix() + except ValueError: + return path.as_posix() + + +def is_accepted_finalization_status(status: object) -> bool: + return status in {"completed", "completed_stable_pending"} + + +def load_checkpoint_records(path: Path) -> dict[str, dict[str, object]]: + if not path.exists(): + return {} + records: dict[str, dict[str, object]] = {} + for line_number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1): + if not line.strip(): + continue + try: + record = json.loads(line) + except json.JSONDecodeError as exc: + raise RuntimeError(f"invalid checkpoint JSON at {path}:{line_number}: {exc}") from exc + if not isinstance(record, dict) or not isinstance(record.get("path"), str): + raise RuntimeError(f"invalid checkpoint record at {path}:{line_number}") + records[str(record["path"])] = record + return records + + +def append_checkpoint_record(path: Path, record: dict[str, object]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a", encoding="utf-8", newline="\n") as stream: + stream.write(json.dumps(record, separators=(",", ":")) + "\n") + stream.flush() + os.fsync(stream.fileno()) + + +def accepted_record_matches_workbook(record: dict[str, object] | None, path: Path) -> bool: + if not record or not is_accepted_finalization_status(record.get("status")): + return False + after_sha256 = record.get("afterSha256") + return isinstance(after_sha256, str) and after_sha256 == sha256_file(path) + + +def seal_terminal_record(path: Path, record: dict[str, object]) -> dict[str, object]: + after = sha256_file(path) + before = str(record["beforeSha256"]) + status = str(record["status"]) + changed = before != after + record["afterSha256"] = after + record["changed"] = changed + if (status == "not_required" or status in PRESERVED_STATUSES) and changed: + raise RuntimeError( + f"{status} violated the byte-preservation boundary for {path}: {before} != {after}" + ) + return record + + +def _finalize_workbook_staged( + path: Path, + excel, + *, + excel_initialization_error: str | None = None, + allow_stable_pending: bool = False, +) -> dict[str, object]: + record: dict[str, object] = { + "path": receipt_workbook_path(path), + "beforeSha256": sha256_file(path), + } + original_bytes: bytes | None = None + try: + inspection = inspect_formula_topology(path) + except Exception as exc: + record.update({ + "status": "preserved_error", + "reason": "package_read_error", + "cacheWriteMode": "none_preserved_original_package", + "packageReadError": str(exc), + }) + return seal_terminal_record(path, record) + + record["formulaCellCount"] = inspection.formula_cell_count + record["formulaTopology"] = inspection.detail + if not inspection.safe: + record.update({ + "status": "preserved_unsupported", + "reason": "unsupported_formula_topology", + "cacheWriteMode": "none_preserved_original_package", + }) + return seal_terminal_record(path, record) + + original_bytes = path.read_bytes() + try: + compatibility_repair = repair_inline_string_cache_conflicts( + path, + expected_sha256=str(record["beforeSha256"]), + ) + record["inlineStringCacheCompatibilityRepair"] = compatibility_repair + except Exception as exc: + record.update({ + "status": "preserved_error", + "reason": "package_compatibility_repair_error", + "cacheWriteMode": "none_preserved_original_package", + "packageCompatibilityRepairError": str(exc), + }) + return seal_terminal_record(path, record) + record["preRefreshSha256"] = sha256_file(path) + + if excel is None: + if original_bytes is not None: + path.write_bytes(original_bytes) + record.update({ + "status": "preserved_error", + "reason": "excel_initialization_error", + "cacheWriteMode": "none_preserved_original_package", + "excelInitializationError": excel_initialization_error or "Excel initialization failed", + }) + return seal_terminal_record(path, record) + + if inspection.formula_cell_count == 0: + workbook = None + try: + workbook = excel.Workbooks.Open( + str(path.resolve()), + UpdateLinks=0, + ReadOnly=True, + IgnoreReadOnlyRecommended=True, + AddToMru=False, + ) + record.update({ + "openMode": "normal_read_only", + "status": "completed", + "reason": "excel_open_verified_no_formula_cells", + "cacheWriteMode": "no_formula_caches", + }) + except Exception as exc: + record.update({ + "status": "preserved_error", + "reason": "excel_com_error", + "cacheWriteMode": "none_preserved_original_package", + "excelError": str(exc), + }) + finally: + if workbook is not None: + workbook.Close(SaveChanges=False) + if str(record["status"]) in PRESERVED_STATUSES and original_bytes is not None: + path.write_bytes(original_bytes) + return seal_terminal_record(path, record) + + try: + refresh_with_excel( + excel, + path, + record, + inspection.formula_cells, + allow_stable_pending=allow_stable_pending, + ) + except ExcelCalculationPending as exc: + record.update({ + "status": "preserved_pending", + "reason": "excel_calculation_state_not_done", + "cacheWriteMode": "none_preserved_original_package", + "calculationStateError": str(exc), + }) + except FormulaCachePatchError as exc: + record.update({ + "status": "preserved_error", + "reason": "package_write_error", + "cacheWriteMode": "none_preserved_original_package", + "packageWriteError": str(exc), + }) + except Exception as exc: + record.update({ + "status": "preserved_error", + "reason": "excel_com_error", + "cacheWriteMode": "none_preserved_original_package", + "excelError": str(exc), + }) + else: + stable_pending = record.get("calculationStateGate") == "accepted_stable_pending" + record.update({ + "status": "completed_stable_pending" if stable_pending else "completed", + "reason": ( + "excel_cache_values_stable_while_calculation_pending" + if stable_pending + else "excel_calculation_done_and_supported_topology" + ), + }) + if str(record["status"]) in PRESERVED_STATUSES and original_bytes is not None: + path.write_bytes(original_bytes) + return seal_terminal_record(path, record) + + +def finalize_workbook( + path: Path, + excel, + *, + excel_initialization_error: str | None = None, + allow_stable_pending: bool = False, + transaction_attempts: int = 3, +) -> dict[str, object]: + """Finalize through a sibling staging file and commit with a source hash check.""" + if transaction_attempts < 1: + raise ValueError("transaction_attempts must be at least 1") + + last_source_change: tuple[str, str] | None = None + for transaction_attempt in range(1, transaction_attempts + 1): + original_bytes = path.read_bytes() + original_sha256 = hashlib.sha256(original_bytes).hexdigest() + descriptor, staging_name = tempfile.mkstemp( + prefix=f".{path.stem}.", + suffix=".finalizing.xlsx", + dir=path.parent, + ) + os.close(descriptor) + staging_path = Path(staging_name) + try: + staging_path.write_bytes(original_bytes) + record = _finalize_workbook_staged( + staging_path, + excel, + excel_initialization_error=excel_initialization_error, + allow_stable_pending=allow_stable_pending, + ) + current_sha256 = sha256_file(path) + if current_sha256 != original_sha256: + last_source_change = (original_sha256, current_sha256) + if transaction_attempt < transaction_attempts: + time.sleep(0.05 * transaction_attempt) + continue + raise RuntimeError( + f"workbook changed during staged finalization for {path}: " + f"{original_sha256} != {current_sha256}" + ) + + record["path"] = receipt_workbook_path(path) + record["beforeSha256"] = original_sha256 + record["transactionMode"] = "staged_compare_and_swap" + record["transactionAttemptCount"] = transaction_attempt + if is_accepted_finalization_status(record.get("status")): + with zipfile.ZipFile(staging_path, "r") as candidate: + bad_entry = candidate.testzip() + if bad_entry: + raise RuntimeError(f"staged workbook has an invalid ZIP entry: {bad_entry}") + os.replace(staging_path, path) + record["afterSha256"] = sha256_file(path) + record["changed"] = record["beforeSha256"] != record["afterSha256"] + return record + finally: + if staging_path.exists(): + staging_path.unlink() + + before, after = last_source_change or ("unknown", "unknown") + raise RuntimeError(f"workbook remained unstable during finalization for {path}: {before} != {after}") + + def main() -> int: parser = argparse.ArgumentParser() source = parser.add_mutually_exclusive_group(required=True) source.add_argument("--dir", help="Root containing *_output.xlsx files") - source.add_argument("--file", help="One candidate .xlsx workbook to refresh before evidence sealing") + source.add_argument("--file", help="One candidate .xlsx workbook to finalize before evidence sealing") parser.add_argument("--receipt", required=True, help="JSON receipt path") - parser.add_argument("--retry-receipt", help="Retry only failed records from this receipt and merge the results") + parser.add_argument("--retry-receipt", help="Retry non-terminal legacy records and merge the results") + parser.add_argument("--checkpoint", help="Durable JSONL progress checkpoint; defaults beside the receipt") + parser.add_argument( + "--allow-stable-pending", + action="store_true", + help="Opt in to cache patching after three identical reads while Excel calculation remains pending", + ) args = parser.parse_args() try: root, files = select_workbooks(args.dir, args.file) except ValueError as exc: parser.error(str(exc)) + all_files = files receipt_path = Path(args.receipt).resolve() + checkpoint_path = ( + Path(args.checkpoint).resolve() + if args.checkpoint + else receipt_path.with_suffix(receipt_path.suffix + ".checkpoint.jsonl") + ) previous_receipt: dict[str, object] | None = None previous_records: dict[str, dict[str, object]] = {} if args.retry_receipt: previous_receipt = json.loads(Path(args.retry_receipt).resolve().read_text(encoding="utf-8")) previous_records = {str(record["path"]): record for record in previous_receipt.get("records", [])} - failed_paths = {path for path, record in previous_records.items() if record.get("status") != "refreshed"} - files = [path for path in files if path.relative_to(Path.cwd()).as_posix() in failed_paths] - if not files: - raise SystemExit(f"no *_output.xlsx files found under {root}") + checkpoint_records = load_checkpoint_records(checkpoint_path) + prior_records = {**previous_records, **checkpoint_records} + selected_paths = {receipt_workbook_path(path) for path in all_files} + prior_records = {path: record for path, record in prior_records.items() if path in selected_paths} + files = [ + path + for path in all_files + if not accepted_record_matches_workbook(prior_records.get(receipt_workbook_path(path)), path) + ] + reused_count = len(all_files) - len(files) + if reused_count: + print(f"reused {reused_count}/{len(all_files)} checkpointed workbook records", flush=True) started = time.time() records: list[dict[str, object]] = [] @@ -661,31 +1256,37 @@ def main() -> int: except Exception as exc: excel_initialization_error = str(exc) for index, path in enumerate(files, start=1): - before = sha256_file(path) - record: dict[str, object] = { - "path": path.relative_to(Path.cwd()).as_posix(), - "beforeSha256": before, - } - try: + record = finalize_workbook( + path, + excel, + excel_initialization_error=excel_initialization_error, + allow_stable_pending=args.allow_stable_pending, + ) + attempt_count = 1 + if record.get("reason") == "excel_com_error": if excel is not None: try: - refresh_with_excel(excel, path, record) - except Exception as excel_error: - record["excelError"] = str(excel_error) - record["openMode"] = "libreoffice_fallback" - refresh_with_libreoffice_cache_patch(path, record) - else: - record["excelInitializationError"] = excel_initialization_error or "Excel initialization failed" - record["openMode"] = "libreoffice_fallback" - refresh_with_libreoffice_cache_patch(path, record) - record["status"] = "refreshed" - except Exception as exc: # Refresh errors must remain visible in the receipt. - record["status"] = "failed" - record["error"] = str(exc) - record["afterSha256"] = sha256_file(path) + excel.Quit() + except Exception: + pass + excel = None + try: + excel, version = create_excel_application() + excel_initialization_error = None + except Exception as exc: + excel_initialization_error = str(exc) + attempt_count = 2 + record = finalize_workbook( + path, + excel, + excel_initialization_error=excel_initialization_error, + allow_stable_pending=args.allow_stable_pending, + ) + record["excelAttemptCount"] = attempt_count records.append(record) + append_checkpoint_record(checkpoint_path, record) if index % 20 == 0 or index == len(files): - print(f"refreshed {index}/{len(files)}", flush=True) + print(f"finalized {index}/{len(files)}", flush=True) finally: if excel is not None: try: @@ -693,17 +1294,23 @@ def main() -> int: except Exception as exc: excel_quit_error = str(exc) - if previous_records: - for record in records: - previous_records[str(record["path"])] = record - records = sorted(previous_records.values(), key=lambda record: str(record["path"])) - failures = [record for record in records if record["status"] != "refreshed"] + merged_records = prior_records + for record in records: + merged_records[str(record["path"])] = record + records = sorted(merged_records.values(), key=lambda record: str(record["path"])) + status_counts = { + status: sum(1 for record in records if record["status"] == status) + for status in FINALIZATION_STATUSES + } + completed_count = status_counts["completed"] + status_counts["completed_stable_pending"] + not_required_count = status_counts["not_required"] + preserved_count = sum(status_counts[status] for status in PRESERVED_STATUSES) attempt_duration_ms = round((time.time() - started) * 1000) previous_duration_ms = int(previous_receipt.get("durationMs", 0)) if previous_receipt else 0 receipt = { "schema": 1, "generatedAt": datetime.now(timezone.utc).isoformat(), - "engine": "Microsoft Excel COM cache extraction with isolated LibreOffice cache fallback", + "engine": "Microsoft Excel COM fail-closed formula cache finalizer", "engineVersion": version, **({"excelInitializationError": excel_initialization_error} if excel_initialization_error else {}), **({"excelQuitError": excel_quit_error} if excel_quit_error else {}), @@ -711,12 +1318,32 @@ def main() -> int: "macros": "disabled", "externalLinkUpdates": "disabled", "calculation": "CalculateFullRebuild", - "persistence": "patch cached formula values into the original OOXML package without saving Excel's formula rewrites", + "calculationStateGate": ( + "state 0, or explicit opt-in with three identical cache reads while pending" + if args.allow_stable_pending + else "both beforeCacheRead and afterCacheRead must equal 0" + ), + "stablePendingOptIn": args.allow_stable_pending, + "formulaTopology": ( + "normal, shared, array, and dataTable caches are patched with byte-identical formula-element proof; " + "unknown types are preserved" + ), + "fallback": "none", + "excelOpen": "normal read-only with repair-mode read-only fallback; Excel output is never saved", + "persistence": "stage and compare-and-swap cached formula values without saving Excel's formula rewrites", + "progressRecovery": "fsync each terminal workbook record to a reusable JSONL checkpoint", }, "root": root.relative_to(Path.cwd()).as_posix(), + "checkpointPath": checkpoint_path.relative_to(Path.cwd()).as_posix(), + "reusedCheckpointCount": reused_count, "workbookCount": len(records), - "refreshedCount": len(records) - len(failures), - "failureCount": len(failures), + "terminalCount": len(records), + "completedCount": completed_count, + "notRequiredCount": not_required_count, + "preservedCount": preserved_count, + "statusCounts": status_counts, + "refreshedCount": completed_count, + "failureCount": len(records) - completed_count, "durationMs": previous_duration_ms + attempt_duration_ms, "lastAttemptDurationMs": attempt_duration_ms, "records": records, @@ -724,7 +1351,7 @@ def main() -> int: receipt_path.parent.mkdir(parents=True, exist_ok=True) receipt_path.write_text(json.dumps(receipt, indent=2) + "\n", encoding="utf-8") print(f"wrote {receipt_path}") - return 1 if failures else 0 + return 0 if __name__ == "__main__": diff --git a/scripts/spreadsheetbench-run-chunked.ts b/scripts/spreadsheetbench-run-chunked.ts index e1c070af..12beb1be 100644 --- a/scripts/spreadsheetbench-run-chunked.ts +++ b/scripts/spreadsheetbench-run-chunked.ts @@ -13,6 +13,7 @@ import type { import type { SpreadsheetBenchTrack } from "../src/eval/spreadsheetBenchAdapter"; import { getModelPricing } from "../src/nodeagent/models/modelCatalog"; import type { OpenRouterFreeModelMode } from "../src/nodeagent/models/openRouterFreeModels"; +import { fingerprintSpreadsheetBenchRunSource } from "../src/eval/spreadsheetBenchRunSourceFingerprint"; const args = process.argv.slice(2); const stageRoot = optionValue("--stage-root"); @@ -39,6 +40,7 @@ const compareStyles = args.includes("--compare-styles"); const compareCharts = args.includes("--compare-charts"); const retryScoreFailures = args.includes("--retry-score-failures"); const refreshExcelCaches = args.includes("--refresh-excel-caches"); +const allowStablePendingCaches = args.includes("--allow-stable-pending-caches"); const clean = args.includes("--clean"); const resume = args.includes("--resume"); const repairMissingModelReceipts = args.includes("--repair-missing-model-receipts"); @@ -57,7 +59,7 @@ const modeRequiresModel = mode === "model-edit-plan" || mode === "nodeagent-work if (!stageRoot || !outputRoot || !jsonOut || !allowedModes.includes(mode) || chunkSize <= 0 || concurrency <= 0 || concurrency > 16) { console.error([ "Usage:", - " npm run benchmark:spreadsheetbench:run-chunked -- --stage-root --output-root --json-out [--mode copy-input-baseline|apply-agent-patch|model-edit-plan|nodeagent-workbook] [--chunk-size 25] [--concurrency 1..16] [--resume] [--repair-missing-model-receipts] [--model ] [--free-auto-mode chat|agent|structured|vision|coding] [--model-batch-size 1] [--model-snapshot-max-cells 800] [--model-snapshot-max-cell-chars 256] [--model-repair-attempts 1] [--refresh-excel-caches] [--task-ids-file ] [--allow-provider-spend --max-provider-cost-usd 1 --provider-call-reserve-usd 0.05] [--clean]", + " npm run benchmark:spreadsheetbench:run-chunked -- --stage-root --output-root --json-out [--mode copy-input-baseline|apply-agent-patch|model-edit-plan|nodeagent-workbook] [--chunk-size 25] [--concurrency 1..16] [--resume] [--repair-missing-model-receipts] [--model ] [--free-auto-mode chat|agent|structured|vision|coding] [--model-batch-size 1] [--model-snapshot-max-cells 800] [--model-snapshot-max-cell-chars 256] [--model-repair-attempts 1] [--refresh-excel-caches] [--allow-stable-pending-caches] [--task-ids-file ] [--allow-provider-spend --max-provider-cost-usd 1 --provider-call-reserve-usd 0.05] [--clean]", "", "Runs staged SpreadsheetBench tasks in fresh child processes and aggregates the reports.", "nodeagent-workbook requires --model, defaults --free-auto-mode to agent, and requires --model-batch-size 1.", @@ -88,6 +90,9 @@ if (modelRepairAttempts < 0 || modelRepairAttempts > 3) throw new Error("--model if (refreshExcelCaches && mode !== "nodeagent-workbook") { throw new Error("--refresh-excel-caches requires --mode nodeagent-workbook."); } +if (allowStablePendingCaches && !refreshExcelCaches) { + throw new Error("--allow-stable-pending-caches requires --refresh-excel-caches."); +} if (providerCallReserveUsd <= 0) throw new Error("--provider-call-reserve-usd must be positive."); if (maxProviderCostUsd !== undefined && maxProviderCostUsd <= 0) throw new Error("--max-provider-cost-usd must be positive."); if (allowProviderSpend && maxProviderCostUsd === undefined) throw new Error("--allow-provider-spend requires --max-provider-cost-usd."); @@ -102,6 +107,9 @@ mkdirSync(dirname(outPath), { recursive: true }); const chunkRoot = resolve(output, ".chunks"); if (!resume) rmSync(chunkRoot, { recursive: true, force: true }); mkdirSync(chunkRoot, { recursive: true }); +const sourceFingerprintPath = resolve(chunkRoot, "source-fingerprint.json"); +const sourceFingerprint = fingerprintSpreadsheetBenchRunSource(); +pinSourceFingerprint(); const stagedTasks = listStagedTasks(); const taskCount = stagedTasks.length; const repairArchive = repairMissingModelReceipts ? collectRepairArchive() : undefined; @@ -154,6 +162,7 @@ async function runChunk(index: number, offset: number, limit: number): Promise(sourceFingerprintPath); + if ( + pinned.schema !== 1 + || pinned.kind !== "spreadsheetbench-run-source-fingerprint" + || pinned.sha256 !== sourceFingerprint.sha256 + || pinned.fileCount !== sourceFingerprint.fileCount + ) { + throw new Error( + `SpreadsheetBench source fingerprint changed since this chunk run began (${String(pinned.sha256)} != ${sourceFingerprint.sha256}). Start a clean run instead of mixing source revisions.`, + ); + } + return; + } + if (resume && hasPriorChunkEvidence()) { + throw new Error( + "SpreadsheetBench resume data has no source fingerprint. Start a clean run instead of adopting unbound chunk evidence.", + ); + } + writeFileSync(sourceFingerprintPath, `${JSON.stringify({ + schema: 1, + kind: "spreadsheetbench-run-source-fingerprint", + generatedAt: new Date().toISOString(), + sha256: sourceFingerprint.sha256, + fileCount: sourceFingerprint.fileCount, + }, null, 2)}\n`); +} + +function assertSourceFingerprint(): void { + const current = fingerprintSpreadsheetBenchRunSource(); + if (current.sha256 !== sourceFingerprint.sha256 || current.fileCount !== sourceFingerprint.fileCount) { + throw new Error( + `SpreadsheetBench source changed during the chunk run (${sourceFingerprint.sha256} != ${current.sha256}). The current chunk is not certifying evidence; restore the source or start a clean run.`, + ); + } +} + +function hasPriorChunkEvidence(): boolean { + return readdirSync(chunkRoot, { withFileTypes: true }).some((entry) => + (entry.isFile() && /^chunk-.*\.json$/.test(entry.name)) + || (entry.isDirectory() && entry.name === "history"), + ); +} + function readReusableChunkReport(path: string, offset: number, limit: number): SpreadsheetBenchRunnerReport | undefined { if (!existsSync(path)) return undefined; try { diff --git a/scripts/spreadsheetbench-run.ts b/scripts/spreadsheetbench-run.ts index caf63d2f..8b8a1d59 100644 --- a/scripts/spreadsheetbench-run.ts +++ b/scripts/spreadsheetbench-run.ts @@ -29,6 +29,7 @@ const compareStyles = args.includes("--compare-styles"); const compareCharts = args.includes("--compare-charts"); const retryScoreFailures = args.includes("--retry-score-failures"); const refreshExcelCaches = args.includes("--refresh-excel-caches"); +const allowStablePendingCaches = args.includes("--allow-stable-pending-caches"); const allowedModes: SpreadsheetBenchRunnerMode[] = [ "copy-input-baseline", @@ -42,7 +43,7 @@ const modeRequiresModel = mode === "model-edit-plan" || mode === "nodeagent-work if (!stageRoot || !outputRoot || !allowedModes.includes(mode)) { console.error([ "Usage:", - " npm run benchmark:spreadsheetbench:run -- --stage-root --output-root [--mode copy-input-baseline|apply-agent-patch|model-edit-plan|nodeagent-workbook] [--model ] [--free-auto-mode chat|agent|structured|vision|coding] [--model-batch-size 1] [--model-snapshot-max-cells 800] [--model-snapshot-max-cell-chars 256] [--model-repair-attempts 1] [--refresh-excel-caches] [--task-ids-file ] [--offset 0] [--limit 3] [--repeats 5] [--retry-failed 2] [--retry-score-failures] [--compare-charts] [--clean] [--json-out ]", + " npm run benchmark:spreadsheetbench:run -- --stage-root --output-root [--mode copy-input-baseline|apply-agent-patch|model-edit-plan|nodeagent-workbook] [--model ] [--free-auto-mode chat|agent|structured|vision|coding] [--model-batch-size 1] [--model-snapshot-max-cells 800] [--model-snapshot-max-cell-chars 256] [--model-repair-attempts 1] [--refresh-excel-caches] [--allow-stable-pending-caches] [--task-ids-file ] [--offset 0] [--limit 3] [--repeats 5] [--retry-failed 2] [--retry-score-failures] [--compare-charts] [--clean] [--json-out ]", "", "copy-input-baseline proves runner/export/scoring plumbing.", "apply-agent-patch reads agent/edit-plan.json, edits the workbook, emits a candidate, then opens evaluator metadata.", @@ -71,6 +72,9 @@ if (mode === "nodeagent-workbook" && modelBatchSize > 1) { if (refreshExcelCaches && mode !== "nodeagent-workbook") { throw new Error("--refresh-excel-caches requires --mode nodeagent-workbook."); } +if (allowStablePendingCaches && !refreshExcelCaches) { + throw new Error("--allow-stable-pending-caches requires --refresh-excel-caches."); +} if (modelSnapshotMaxCells !== undefined && modelSnapshotMaxCells < 1) { throw new Error("--model-snapshot-max-cells must be at least 1."); } @@ -99,6 +103,7 @@ const report = await runStagedSpreadsheetBench({ modelSnapshotMaxCellChars, modelRepairAttempts, refreshExcelCaches, + allowStablePendingCaches, taskIds, limit, offset, diff --git a/scripts/spreadsheetbench-structural-repair.ps1 b/scripts/spreadsheetbench-structural-repair.ps1 new file mode 100644 index 00000000..db11cfca --- /dev/null +++ b/scripts/spreadsheetbench-structural-repair.ps1 @@ -0,0 +1,126 @@ +param( + [Parameter(Mandatory = $true)][string]$WorkbookPath, + [Parameter(Mandatory = $true)][string]$PlanPath +) + +$ErrorActionPreference = 'Stop' +$plan = Get-Content -LiteralPath $PlanPath -Raw | ConvertFrom-Json +if ($plan.schema -ne 1 -or @($plan.repairs).Count -lt 1) { + throw 'Structural repair plan must contain at least one schema-1 repair.' +} + +$excel = $null +$workbook = $null +$insertedRows = 0 +$formulaReplacements = 0 +$explicitFormulaRepairs = 0 +$repairIds = @() +$calculationPasses = 6 + +try { + $excel = New-Object -ComObject Excel.Application + $excel.Visible = $false + $excel.DisplayAlerts = $false + $excel.AskToUpdateLinks = $false + $excel.EnableEvents = $false + $excel.AutomationSecurity = 3 + $workbook = $excel.Workbooks.Open((Resolve-Path -LiteralPath $WorkbookPath).Path, 0, $false) + + foreach ($repair in @($plan.repairs)) { + if ($repair.status -ne 'complete' -or $repair.basis -ne 'visible_workbook_invariants') { + throw "Repair $($repair.repairId) is not a complete visible-invariant contract." + } + if ($repair.kind -ne 'insert_missing_selector_row') { + throw "Unsupported structural repair kind $($repair.kind)." + } + $sheet = $workbook.Worksheets.Item([string]$repair.sheet) + try { + $row = $sheet.Rows.Item([int]$repair.insertRow) + try { [void]$row.Insert(-4121) } finally { [void][Runtime.InteropServices.Marshal]::ReleaseComObject($row) } + $insertedRows += 1 + $sheet.Range([string]$repair.labelCell).Value2 = [string]$repair.label + $sheet.Range([string]$repair.selectorCell).Value2 = [double]$repair.selectorValue + + $replaced = 0 + $used = $sheet.UsedRange + try { + foreach ($cell in $used.Cells) { + try { + if ($cell.HasFormula -and ([string]$cell.Formula).Contains([string]$repair.formulaSearch)) { + $cell.Formula = ([string]$cell.Formula).Replace( + [string]$repair.formulaSearch, + [string]$repair.formulaReplace + ) + $replaced += 1 + } + } finally { + [void][Runtime.InteropServices.Marshal]::ReleaseComObject($cell) + } + } + } finally { + [void][Runtime.InteropServices.Marshal]::ReleaseComObject($used) + } + if ($replaced -ne [int]$repair.expectedFormulaReplacementCount) { + throw "Repair $($repair.repairId) expected $($repair.expectedFormulaReplacementCount) formula replacements but applied $replaced." + } + $formulaReplacements += $replaced + + foreach ($formulaRepair in @($repair.formulaRepairs)) { + $formulaSheet = $workbook.Worksheets.Item([string]$formulaRepair.sheet) + try { + $formulaSheet.Range([string]$formulaRepair.cell).Formula = [string]$formulaRepair.formula + $explicitFormulaRepairs += 1 + } finally { + [void][Runtime.InteropServices.Marshal]::ReleaseComObject($formulaSheet) + } + } + $repairIds += [string]$repair.repairId + } finally { + [void][Runtime.InteropServices.Marshal]::ReleaseComObject($sheet) + } + } + + $excel.Calculation = -4105 + $excel.Iteration = $true + $excel.MaxIterations = 1000 + $excel.MaxChange = 0.000001 + $workbook.ForceFullCalculation = $true + foreach ($sheet in $workbook.Worksheets) { + try { + $sheet.EnableCalculation = $false + $sheet.EnableCalculation = $true + try { $sheet.UsedRange.Dirty() } catch {} + } finally { + [void][Runtime.InteropServices.Marshal]::ReleaseComObject($sheet) + } + } + for ($pass = 0; $pass -lt $calculationPasses; $pass += 1) { + $excel.CalculateFullRebuild() + } + [void]$workbook.Save() + [void]$workbook.Close($true) + $workbook = $null +} finally { + if ($workbook -ne $null) { + try { [void]$workbook.Close($false) } catch {} + } + if ($excel -ne $null) { + try { [void]$excel.Quit() } catch {} + } + if ($workbook -ne $null) { [void][Runtime.InteropServices.Marshal]::ReleaseComObject($workbook) } + if ($excel -ne $null) { [void][Runtime.InteropServices.Marshal]::ReleaseComObject($excel) } + [GC]::Collect() + [GC]::WaitForPendingFinalizers() +} + +[pscustomobject]@{ + schema = 1 + backend = 'excel_com' + status = 'completed' + workbookPath = (Resolve-Path -LiteralPath $WorkbookPath).Path + repairIds = @($repairIds) + insertedRowCount = $insertedRows + formulaReplacementCount = $formulaReplacements + explicitFormulaRepairCount = $explicitFormulaRepairs + calculationPasses = $calculationPasses +} | ConvertTo-Json -Compress diff --git a/src/app/store.tsx b/src/app/store.tsx index 066a5178..f52fc042 100644 --- a/src/app/store.tsx +++ b/src/app/store.tsx @@ -47,6 +47,9 @@ export type { OfflineQueueSnapshot } from "../notifications/offlineQueue"; const VARIANCE: Record = { r_rev: "+24%", r_cogs: "+27.5%", r_gp: "+21.7%", r_ni: "+22.4%" }; export type EditFeedback = { ok: boolean; reason?: string; version?: number }; +export type ArtifactEditBundleFeedback = + | { ok: true; artifactVersion: number; results: Array<{ opId: string; elementId: string; version: number }> } + | { ok: false; reason: string; opId?: string; elementId?: string; expected?: number; actual?: number }; type UndoEntry = { roomId: string; op: ChangeOp }; export type AgentCostKind = "exact" | "estimated"; export type AgentRunTelemetry = { model: string; steps: number; toolCalls: number; inputTokens: number; outputTokens: number; costUsd: number; costKind: AgentCostKind; ms: number }; @@ -75,6 +78,13 @@ export function projectAgentRunTelemetry(run: PersistedAgentRunTelemetry): Agent } export type AgentJobTelemetry = { id: string; + artifactId?: string; + request?: { + references?: Array<{ id?: string; title?: string; kind?: string }>; + allowedElementIds?: string[]; + mutationScope?: string; + targetArtifactId?: string; + }; goal?: string; status: string; entrypoint?: string; @@ -155,7 +165,7 @@ export function selectAgentRunTelemetryForJob( /** Shape of a free-auto agent job row from the convex jobs subscription (used by lastLongFreeJob + activeLongFreeJobs). */ type FreeJobRow = { - _id: string; goal?: string; status: string; entrypoint?: string; scope?: string; runtime?: string; attempts: number; maxAttempts: number; + _id: string; artifactId?: string; request?: AgentJobTelemetry["request"]; goal?: string; status: string; entrypoint?: string; scope?: string; runtime?: string; attempts: number; maxAttempts: number; runtimeProfile?: AgentRuntimeProfile; modelPolicy: string; approvalPolicy?: string; evidencePolicy?: string; handoff?: { reason?: string }; nextRunAt?: number; finalText?: string; error?: string; latestRunId?: string; actionSliceCount?: number; queryCount?: number; mutationCount?: number; modelCallCount?: number; toolCallCount?: number; costUsd?: number; costKind?: AgentCostKind; schedulerHandoffCount?: number; receiptCount?: number; createdAt?: number; updatedAt: number; @@ -167,6 +177,8 @@ function isActiveFreeJob(status: string): boolean { function mapConvexFreeJob(j: FreeJobRow): AgentJobTelemetry { return { id: String(j._id), + artifactId: j.artifactId ? String(j.artifactId) : undefined, + request: j.request, goal: j.goal, status: j.status, entrypoint: j.entrypoint, @@ -277,6 +289,8 @@ export type AgentAskInput = { maxAttempts?: number; /** Credit/depth mode for the run (Quick/Standard/Deep). Defaults to the store's selected mode. */ mode?: AgentCreditMode; + /** Follow-up handling when this requester already has a public run in flight. */ + disposition?: "start" | "queue" | "redirect"; }; export type ActorProof = { actor: Actor; token: string }; export type PrivateStreamAccess = { requester: ActorProof; driven: boolean }; @@ -391,6 +405,8 @@ export interface RoomStore { awareness(roomId: string, agentId?: string): { activeLocks: Lock[] }; /** Apply a hand edit (CAS). Returns feedback so the UI can surface a conflict honestly. */ applyEdit(args: { roomId: string; op: ChangeOp; actor: Actor }): Promise; + /** Apply a bounded same-artifact edit bundle atomically. A rejected bundle applies no elements. */ + applyArtifactEdits(args: { roomId: string; artifactId: string; ops: ChangeOp[]; actor: Actor }): Promise; /** Execute a bounded read-only notebook kernel. Persistence remains an ordinary artifact CAS write. */ executeNotebookKernel(args: { roomId: string; request: NotebookKernelRequest }): Promise; /** Offline edit-hold: live-mode CAS edits that failed on a TRANSPORT error (not a server answer) @@ -990,6 +1006,16 @@ export function EngineStoreProvider({ roomId, children }: { roomId: string; me: if (r.ok) pushUndo(undoStack.current, withAppliedVersion(undo, r.toVersion)); return r.ok ? { ok: true, version: r.toVersion } : { ok: false, reason: r.reason }; }, + applyArtifactEdits: async (args) => { + const artifact = engine.getArtifact(args.artifactId); + const undos = args.ops.map((op) => makeUndoEntry(args.roomId, artifact, op)); + const result = engine.applyArtifactEdits(args); + if (!result.ok) return result; + result.results.forEach((entry, index) => { + pushUndo(undoStack.current, withAppliedVersion(undos[index], entry.version)); + }); + return result; + }, executeNotebookKernel: async ({ request }) => executeNotebookKernel(request, { backend: "memory", now: Date.now() }), canUndo: (id) => (undoStack.current.get(id)?.length ?? 0) > 0, undoLastEdit: async (id, actor) => { @@ -1656,6 +1682,27 @@ export function ConvexStoreProvider({ roomId, me, proof, children }: { roomId: s local.setQuery(api.artifacts.versions, versionsQ, curVersions.map((a) => String(a.id) === String(args.artifactId) ? { ...a, order, version: a.version + 1, updatedAt: Date.now() } : a) as typeof curVersions); } }); + const applyArtifactEditsMutation = useMutation(api.artifacts.applyArtifactEdits).withOptimisticUpdate((local, args) => { + const elementsQ = { roomId: args.roomId, artifactId: args.artifactId, requester: args.requester }; + const curEls = local.getQuery(api.artifacts.elements, elementsQ); + if (curEls === undefined) return; + const versionsQ = { roomId: args.roomId, requester: args.requester }; + const curVersions = local.getQuery(api.artifacts.versions, versionsQ); + const rowVer = curVersions?.find((artifact) => String(artifact.id) === String(args.artifactId)); + let elements = elementsPayloadToMap(curEls); + let order = (rowVer?.order ?? Object.keys(elements)) as string[]; + for (const edit of args.edits) { + ({ elements, order } = applyCellToElements(elements, order, edit.elementId, edit.kind, edit.value, args.requester.actor)); + } + local.setQuery(api.artifacts.elements, elementsQ, elementsMapToPayload(elements, curEls) as typeof curEls); + if (curVersions && rowVer) { + local.setQuery(api.artifacts.versions, versionsQ, curVersions.map((artifact) => ( + String(artifact.id) === String(args.artifactId) + ? { ...artifact, order, version: artifact.version + args.edits.length, updatedAt: Date.now() } + : artifact + )) as typeof curVersions); + } + }); // ── Offline edit-hold (Latency: "offline edits held, visible, never lost") ── // TRANSPORT failures (fetch/WebSocket down) hold the CAS op in a bounded in-memory + // localStorage queue and replay it on reconnect through the SAME applyEdit path, so a @@ -1994,7 +2041,7 @@ export function ConvexStoreProvider({ roomId, me, proof, children }: { roomId: s const sessions = (data?.sessions ?? []) as unknown as AgentSession[]; const drafts = (data?.drafts ?? []) as unknown as Draft[]; const isHost = members.some((m) => m.id === me.id && m.role === "host"); - const reshapeMsgs = (rows: typeof pub): Message[] => rows.map((m: { _id: string; roomId: string; channel: string; author: Actor; text: string; clientMsgId: string; kind: Message["kind"]; createdAt: number; streamId?: string }) => ({ id: m._id as string, roomId: m.roomId as string, channel: m.channel === "public" ? "public" : { private: m.channel }, author: m.author as Actor, text: m.text, clientMsgId: m.clientMsgId, kind: m.kind, createdAt: m.createdAt, streamId: m.streamId })); + const reshapeMsgs = (rows: typeof pub): Message[] => rows.map((m: { _id: string; roomId: string; channel: string; author: Actor; text: string; clientMsgId: string; kind: Message["kind"]; jobId?: string; createdAt: number; streamId?: string }) => ({ id: m._id as string, roomId: m.roomId as string, channel: m.channel === "public" ? "public" : { private: m.channel }, author: m.author as Actor, text: m.text, clientMsgId: m.clientMsgId, kind: m.kind, jobId: m.jobId ? String(m.jobId) : undefined, createdAt: m.createdAt, streamId: m.streamId })); const allTraces = (traces as { _id: string; roomId: string; ts: number; actor: Actor; type: string; summary: string; detail?: string }[]).map((t) => ({ id: t._id, roomId: t.roomId, ts: t.ts, actor: t.actor, type: t.type as TraceEvent["type"], summary: t.summary, detail: t.detail })); // Hoisted sheet resolution — shared by addActivityToSheet (researchActivity resolves server-side). const researchSheet = metaArtifacts.find((a) => (a as { kind?: string }).kind === "sheet" && (a as { title?: string }).title === "Company research"); @@ -2091,6 +2138,44 @@ export function ConvexStoreProvider({ roomId, me, proof, children }: { roomId: s return { ok: false, reason: e instanceof Error ? e.message : "edit_failed" }; } }, + applyArtifactEdits: async ({ artifactId, ops }) => { + const artifact = artifacts.find((candidate) => candidate.id === artifactId); + const undos = ops.map((op) => makeUndoEntry(roomId, artifact, op)); + try { + const result = await applyArtifactEditsMutation({ + roomId: rid, + artifactId: artifactId as never, + edits: ops.map((op) => ({ + opId: op.opId, + elementId: op.elementId, + kind: op.kind, + value: op.kind === "delete" ? null : op.value, + baseVersion: op.baseVersion, + })), + requester: proof, + }); + result.results.forEach((entry, index) => { + pushUndo(undoStack.current, withAppliedVersion(undos[index], entry.version)); + }); + return result; + } catch (error) { + const data = (error as { data?: unknown }).data; + if (data && typeof data === "object") { + const failure = data as { reason?: unknown; opId?: unknown; elementId?: unknown; expected?: unknown; actual?: unknown }; + if (typeof failure.reason === "string") { + return { + ok: false, + reason: failure.reason, + ...(typeof failure.opId === "string" ? { opId: failure.opId } : {}), + ...(typeof failure.elementId === "string" ? { elementId: failure.elementId } : {}), + ...(typeof failure.expected === "number" ? { expected: failure.expected } : {}), + ...(typeof failure.actual === "number" ? { actual: failure.actual } : {}), + }; + } + } + return { ok: false, reason: isNetworkError(error) ? "offline_bundle_not_applied" : error instanceof Error ? error.message : "artifact_edit_bundle_failed" }; + } + }, executeNotebookKernel: async ({ request }) => runNotebookKernel({ roomId: rid, requester: proof, kind: request.kind, input: request.input, tables: request.tables }), offlineEditQueue: () => offlineSnap, acknowledgeOfflineConflicts: () => { @@ -2215,6 +2300,7 @@ export function ConvexStoreProvider({ roomId, me, proof, children }: { roomId: s contextArtifactId: input.contextArtifactId, contextArtifactRequired: input.contextArtifactRequired, allowedElementIds: input.allowedElementIds, + disposition: input.disposition, goal: withReferenceContext(input.goal, references), }); selectLongJob(String(started.jobId)); @@ -2463,7 +2549,7 @@ export function ConvexStoreProvider({ roomId, me, proof, children }: { roomId: s return result.rowId ? { artifactId: targetArt.id as string, rowId: result.rowId as string, created: result.created } : undefined; }, }; - }, [data, metaArtifacts, elementsByArtifact, presenceByArtifact, pub, priv, traces, okfLens, runs, jobs, selectedLongJobRow, jobAttempts, jobDetail, proposals, passiveActivity, mergedCaptures, applyCellEdit, applyEditCore, offlineQueue, offlineSnap, scheduleOfflineReplay, sendMsg, toggle, editMsg, resolveProposalMutation, addResearchRowsMutation, ensurePassiveResearchRowMutation, createArtifactMutation, uploadSourceFile, runSemanticConflictDrillMutation, runAgent, runPrivateAgent, runNotebookKernel, createPrivateReplyStream, startAgentJob, startPublicAskJob, selectLongJob, updatePresenceMutation, clearPresenceMutation, cancelFreeAutoJob, retryFreeAutoJob, dismissActivityMutation, researchActivityMutation, practiceActivityMutation, creditMode, creditBalanceQ, creditUsageQ, rid, roomId, proof, me.id, me.name]); + }, [data, metaArtifacts, elementsByArtifact, presenceByArtifact, pub, priv, traces, okfLens, runs, jobs, selectedLongJobRow, jobAttempts, jobDetail, proposals, passiveActivity, mergedCaptures, applyCellEdit, applyArtifactEditsMutation, applyEditCore, offlineQueue, offlineSnap, scheduleOfflineReplay, sendMsg, toggle, editMsg, resolveProposalMutation, addResearchRowsMutation, ensurePassiveResearchRowMutation, createArtifactMutation, uploadSourceFile, runSemanticConflictDrillMutation, runAgent, runPrivateAgent, runNotebookKernel, createPrivateReplyStream, startAgentJob, startPublicAskJob, selectLongJob, updatePresenceMutation, clearPresenceMutation, cancelFreeAutoJob, retryFreeAutoJob, dismissActivityMutation, researchActivityMutation, practiceActivityMutation, creditMode, creditBalanceQ, creditUsageQ, rid, roomId, proof, me.id, me.name]); // E2E test seam: expose runCollab/runSemanticConflictDrill via window so tests can trigger // collaboration and conflict drills without the removed CollabBar buttons. diff --git a/src/app/styles.css b/src/app/styles.css index 6774f98b..325044e6 100644 --- a/src/app/styles.css +++ b/src/app/styles.css @@ -804,12 +804,24 @@ button.r-offline-pill:hover, button.r-offline-pill:focus-visible { border-color: .r-agent-stream { min-height: auto; } } .r-private-banner { padding: 8px 12px; background: var(--accent-tint); border-bottom: 1px solid var(--accent-border); display: flex; align-items: center; gap: var(--space-2); font-size: 11px; color: var(--accent-ink); font-weight: 600; } -.r-job-strip { padding: var(--space-2) 12px; border-bottom: 1px solid var(--line); display: flex; align-items: center; gap: 8px; min-height: 30px; font-size: 11px; color: var(--text-muted); background: rgba(124,135,148,.08); overflow: hidden; white-space: nowrap; text-overflow: ellipsis; } -.r-job-strip > svg, .r-job-strip > button, .r-job-strip > .r-tag { flex: none; } -.r-job-route { min-width: 0; flex: 1; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.r-job-strip { padding: 6px 12px; border-bottom: 1px solid var(--line); display: grid; gap: 4px; min-height: 30px; font-size: 11px; color: var(--text-muted); background: rgba(124,135,148,.08); overflow: hidden; } +.r-job-strip-main { display: flex; align-items: center; gap: 7px; min-width: 0; } +.r-job-strip-main > svg, .r-job-strip-main > button, .r-job-strip-main > .r-tag { flex: none; } +.r-job-route { display: none; } +.r-job-phase { min-width: 0; flex: 1; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; color: var(--text-secondary); font-weight: 650; text-transform: capitalize; } +.r-job-elapsed { flex: none; color: var(--text-tertiary); font-variant-numeric: tabular-nums; } +.r-job-scope { display: grid; grid-template-columns: minmax(0, 1fr) minmax(0, 1fr) minmax(0, 1fr) auto; align-items: center; gap: 6px 12px; min-width: 0; color: var(--text-tertiary); } +.r-job-scope span { min-width: 0; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.r-job-scope b { margin-right: 4px; color: var(--text-muted); font-weight: 650; } +.r-job-scope button { display: inline-flex; align-items: center; gap: 3px; border: 0; background: transparent; color: var(--accent-ink); padding: 1px 0; font: inherit; font-weight: 650; cursor: pointer; } +.r-job-scope button:hover { color: var(--text-primary); } .r-job-detail-toggle { margin-left: auto; display: inline-flex; align-items: center; gap: 3px; border: 1px solid var(--line); background: var(--bg-secondary); color: var(--text-muted); border-radius: 6px; padding: 2px 6px; font-size: 10.5px; font-weight: 600; cursor: pointer; } .r-job-detail-toggle:hover { color: var(--text-primary); border-color: var(--line-strong); } .r-job-detail { border-bottom: 1px solid var(--line); background: rgba(124,135,148,.06); padding: 8px 12px; display: grid; gap: 8px; font-size: 11px; color: var(--text-muted); } +@media (max-width: 760px) { + .r-job-scope { grid-template-columns: minmax(0, 1fr) auto; } + .r-job-scope span:nth-child(2), .r-job-scope span:nth-child(3) { display: none; } +} .r-job-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(140px, 1fr)); gap: 5px 12px; } .r-job-grid span { color: var(--text-tertiary); } .r-job-grid b { color: var(--text-secondary); font-weight: 600; min-width: 0; overflow-wrap: anywhere; } diff --git a/src/engine/roomEngine.ts b/src/engine/roomEngine.ts index 511f85eb..516fb9d6 100644 --- a/src/engine/roomEngine.ts +++ b/src/engine/roomEngine.ts @@ -52,6 +52,27 @@ export interface MergeOutcome { semantic?: { conflictId: string; resolution: SemanticResolution; proposalIds: string[] }; } +export type ArtifactEditBundleResult = + | { + ok: true; + artifactVersion: number; + results: Array<{ + opId: string; + elementId: string; + version: number; + }>; + } + | { + ok: false; + reason: string; + opId?: string; + elementId?: string; + expected?: number; + actual?: number; + by?: Actor; + lockId?: string; + }; + function stableValueKey(value: unknown): string { try { return JSON.stringify(value); } catch { return String(value); } @@ -409,6 +430,168 @@ export class RoomEngine { } /** Raw CAS apply — used by applyEdit, proposal approval, and merge. */ + /** Apply a bounded set of edits as one in-memory transaction. */ + applyArtifactEdits(args: { + roomId: string; + artifactId: string; + ops: ChangeOp[]; + actor: Actor; + }): ArtifactEditBundleResult { + const { roomId, artifactId, ops, actor } = args; + if (!Array.isArray(ops) || ops.length === 0 || ops.length > 64) { + return { ok: false, reason: "invalid" }; + } + if ( + typeof roomId !== "string" + || !roomId.trim() + || typeof artifactId !== "string" + || !artifactId.trim() + || !actor + || (actor.kind !== "user" && actor.kind !== "agent") + || typeof actor.id !== "string" + || !actor.id.trim() + || typeof actor.name !== "string" + || !actor.name.trim() + ) { + return { ok: false, reason: "invalid" }; + } + + const room = this.rooms.get(roomId); + const artifact = this.artifacts.get(artifactId); + if (!room || !artifact || artifact.roomId !== roomId) { + return { ok: false, reason: "not_found" }; + } + if (actor.kind === "agent" && !room.autoAllow) { + return { ok: false, reason: "pending_approval" }; + } + + const opIds = new Set(); + const elementIds = new Set(); + const lockCheckAt = this.now(); + for (const op of ops) { + if ( + !op + || typeof op.opId !== "string" + || !op.opId.trim() + || typeof op.artifactId !== "string" + || op.artifactId !== artifactId + || typeof op.elementId !== "string" + || !op.elementId.trim() + || !(["set", "create", "delete"] as const).includes(op.kind) + || !Number.isSafeInteger(op.baseVersion) + || op.baseVersion < 0 + || ((op.kind === "set" || op.kind === "create") && op.value === undefined) + ) { + return { ok: false, reason: "invalid", opId: op?.opId, elementId: op?.elementId }; + } + if (opIds.has(op.opId) || this.appliedOps.has(op.opId)) { + return { ok: false, reason: "duplicate", opId: op.opId, elementId: op.elementId }; + } + if (elementIds.has(op.elementId)) { + return { ok: false, reason: "duplicate_element", opId: op.opId, elementId: op.elementId }; + } + opIds.add(op.opId); + elementIds.add(op.elementId); + + const lock = [...this.locks.values()].find( + (candidate) => candidate.status === "active" + && (candidate.expiresAt === undefined || candidate.expiresAt > lockCheckAt) + && candidate.artifactId === artifactId + && candidate.elementIds.includes(op.elementId), + ); + if (lock && !sameActor(lock.holder, actor)) { + return { + ok: false, + reason: "locked", + opId: op.opId, + elementId: op.elementId, + by: lock.holder, + lockId: lock.id, + }; + } + + if (actor.kind === "agent" && (op.kind === "set" || op.kind === "create")) { + const declared = artifact.meta?.dataframe?.columns; + const column = declared && declared.length ? columnIdOfElement(op.elementId) : null; + if (column && !declared!.some((candidate) => candidate.id === column)) { + return { ok: false, reason: "no_such_column", opId: op.opId, elementId: op.elementId }; + } + } + + const element = artifact.elements[op.elementId]; + if (op.kind === "create") { + if (element) return { ok: false, reason: "duplicate", opId: op.opId, elementId: op.elementId }; + continue; + } + if (!element) return { ok: false, reason: "not_found", opId: op.opId, elementId: op.elementId }; + if (element.version !== op.baseVersion) { + return { + ok: false, + reason: "conflict", + opId: op.opId, + elementId: op.elementId, + expected: op.baseVersion, + actual: element.version, + }; + } + if (op.kind === "set" && actor.kind === "agent" && formulaOf(element.value) && !formulaOf(op.value)) { + return { ok: false, reason: "formula_protected", opId: op.opId, elementId: op.elementId }; + } + } + + const artifactSnapshot: Artifact = { + ...artifact, + elements: Object.fromEntries( + Object.entries(artifact.elements).map(([id, element]) => [id, { ...element }]), + ), + order: [...artifact.order], + }; + const tracesSnapshot = [...this.traces]; + const appliedOpsSnapshot = new Set(this.appliedOps); + const idCounterSnapshot = this.idc; + const restore = () => { + this.artifacts.set(artifactId, artifactSnapshot); + this.traces = tracesSnapshot; + this.appliedOps = appliedOpsSnapshot; + this.idc = idCounterSnapshot; + }; + + const results: Array<{ + opId: string; + elementId: string; + version: number; + }> = []; + try { + for (const op of ops) { + const result = this.applyOpInternal(op, actor); + if (!result.ok) { + restore(); + return { + ok: false, + reason: result.reason, + opId: op.opId, + elementId: op.elementId, + ...(result.reason === "conflict" + ? { expected: result.expected, actual: result.actual } + : {}), + ...(result.reason === "locked" ? { by: result.by, lockId: result.lockId } : {}), + }; + } + results.push({ + opId: op.opId, + elementId: op.elementId, + version: result.toVersion, + }); + } + } catch { + restore(); + return { ok: false, reason: "internal_error" }; + } + + this.emit(); + return { ok: true, artifactVersion: artifact.version, results }; + } + private applyOpInternal(op: ChangeOp, actor: Actor): EditResult { const art = this.artifacts.get(op.artifactId); if (!art) return { ok: false, reason: "not_found" }; diff --git a/src/engine/types.ts b/src/engine/types.ts index 82f88c3d..be51c849 100644 --- a/src/engine/types.ts +++ b/src/engine/types.ts @@ -89,6 +89,8 @@ export interface CellPayload { confidence?: number; formula?: string; numFmt?: string; + /** Canonical uppercase AARRGGBB. Omission preserves the current font color. */ + fontColor?: string; error?: string; normalizedValue?: unknown; attempts?: number; @@ -407,6 +409,8 @@ export interface Message { text: string; clientMsgId: string; // idempotency + optimistic-reconcile key kind: "chat" | "agent" | "system"; + /** Durable public-agent run correlation. Legacy/local messages may omit it. */ + jobId?: string; toolParts?: ToolPart[]; createdAt: number; /** persistent-text-streaming id (live mode): body streams from the component while text is diff --git a/src/eval/spreadsheetBenchNodeAgentBridge.ts b/src/eval/spreadsheetBenchNodeAgentBridge.ts index ede85fb2..110a7e11 100644 --- a/src/eval/spreadsheetBenchNodeAgentBridge.ts +++ b/src/eval/spreadsheetBenchNodeAgentBridge.ts @@ -2,6 +2,12 @@ import { createHash } from "node:crypto"; import { existsSync, mkdirSync, readFileSync, realpathSync } from "node:fs"; import { basename, dirname, isAbsolute, relative, resolve } from "node:path"; import ExcelJS from "exceljs"; +import { + normalizeSpreadsheetFontColor, + resolveSpreadsheetFontColor, + spreadsheetThemeIndexForColor, + type SpreadsheetFontColorSource, +} from "../shared/spreadsheetFontColor"; import type { SpreadsheetBenchTrack } from "./spreadsheetBenchAdapter"; import { readSpreadsheetBenchWorkbookForMutation } from "./spreadsheetBenchScorer"; import { @@ -32,14 +38,23 @@ import type { import { MANAGED_LOCK_SYSTEM_PROMPT } from "../nodeagent/models/prompts/systemPrompt"; import { EXECUTE_VERIFIED_WORKBOOK_PLAN_TOOL, + EXECUTE_WORKBOOK_STRUCTURE_REPAIR_TOOL_NAME, PRODUCTION_ROOM_TOOLS, } from "../nodeagent/skills/spreadsheet/cellMutator"; import { + applySpreadsheetBenchStructuralRepairs, + detectSpreadsheetBenchStructuralRepair, + type SpreadsheetBenchStructuralRepairPlan, + type SpreadsheetBenchStructuralRepairReceipt, +} from "./spreadsheetBenchStructuralRepair"; +import { + buildWorkbookSuggestedPlan, extractWorkbookTaskReferences, inspectWorkbookTask, normalizeAddress, normalizeFormula, selectWorkbookTaskCells, + workbookCellKey, type WorkbookObservedCell, type WorkbookTaskInspection, } from "../nodeagent/skills/spreadsheet/workbookTaskIntelligence"; @@ -61,6 +76,7 @@ export const SPREADSHEETBENCH_NODEAGENT_BRIDGE_SCHEMA = "noderoom.spreadsheetben const BRIDGE_TOOL_NAMES = [ "inspect_workbook", + "execute_workbook_structure_repair", "execute_verified_workbook_plan", "verify_workbook", "list_artifacts", @@ -73,7 +89,11 @@ const BRIDGE_TOOL_NAMES = [ const EXECUTE_VERIFIED_WORKBOOK_PLAN_TOOL_NAME = "execute_verified_workbook_plan"; const WRITE_TOOL_NAMES = new Set(["write_locked_cell", "write_locked_cells"]); -const MUTATION_TOOL_NAMES = new Set([...WRITE_TOOL_NAMES, EXECUTE_VERIFIED_WORKBOOK_PLAN_TOOL_NAME]); +const COMPOSITE_WORKBOOK_TOOL_NAMES = new Set([ + EXECUTE_VERIFIED_WORKBOOK_PLAN_TOOL_NAME, + EXECUTE_WORKBOOK_STRUCTURE_REPAIR_TOOL_NAME, +]); +const MUTATION_TOOL_NAMES = new Set([...WRITE_TOOL_NAMES, ...COMPOSITE_WORKBOOK_TOOL_NAMES]); const READ_REFERENCE_RE = /^(?:(?:'((?:[^']|'')+)'|([^!]+))!\s*)?(\$?[A-Z]{1,3}\$?[1-9][0-9]*)(?:\s*:\s*(\$?[A-Z]{1,3}\$?[1-9][0-9]*))?$/i; const EXCEL_MAX_ROW = 1_048_576; const EXCEL_MAX_COLUMN = 16_384; @@ -183,6 +203,7 @@ export type SpreadsheetBenchNodeAgentBridgeReceipt = { usage: TokenUsage; }; recalculation: SpreadsheetBenchNodeAgentRecalculationReceipt; + structuralRepair?: SpreadsheetBenchStructuralRepairReceipt; candidateFinalization?: SpreadsheetBenchCandidateFinalizationReceipt; frame: ReasoningFrameRunReceipt; trace: NodeAgentTrace; @@ -199,7 +220,16 @@ export type RunSpreadsheetBenchNodeAgentBridgeOptions = { modelTimeoutMs?: number; snapshotMaxCells?: number; scanMaxCells?: number; + /** + * Keep ambiguous audit tasks evidence-bounded: inspect once, then stop without + * provider spend when the visible workbook exposes no safe write contract. + */ + boundedAuditPlanning?: boolean; now?: () => number; + applyStructuralRepairs?: (args: { + workbookPath: string; + repairs: SpreadsheetBenchStructuralRepairPlan[]; + }) => SpreadsheetBenchStructuralRepairReceipt | undefined | Promise; finalizeCandidate?: (args: { taskId: string; track: SpreadsheetBenchTrack; @@ -259,8 +289,25 @@ export async function runSpreadsheetBenchNodeAgentBridge( sourceHash, startedAt, }); - const frame = buildBridgeFrame(task.manifest, room.artifactIds(), traceId); + const boundedAuditPlanning = options.boundedAuditPlanning !== false; + const initialInspection = room.taskInspection(); + const boundedNoEvidenceAudit = boundedAuditPlanning + && bridgeInspectionNeedsBoundedNoEvidenceCompletion(initialInspection); + const frame = buildBridgeFrame( + task.manifest, + room.artifactIds(), + traceId, + boundedNoEvidenceAudit + ? "Inspect visible workbook evidence and report only whether a safe mutation contract is available; do not create, edit, write, update, fill, set, delete, commit, or apply cells when none is present." + : undefined, + ); const workflowController = new BridgeWorkbookWorkflowController(intelligenceInstruction, room.artifactIds()[0]); + const effectiveModel = contractFirstBridgeModel( + options.model, + room, + intelligenceInstruction, + boundedAuditPlanning, + ); const tools = selectBridgeTools( [...PRODUCTION_ROOM_TOOLS, EXECUTE_VERIFIED_WORKBOOK_PLAN_TOOL], workflowController, @@ -268,11 +315,16 @@ export async function runSpreadsheetBenchNodeAgentBridge( const frameReceipt = await runReasoningFrame({ rt: room, frame, - model: options.model, + model: effectiveModel, tools, maxSteps: Math.max(1, Math.trunc(options.maxSteps ?? 18)), deadlineAt: options.modelTimeoutMs === undefined ? undefined : startedAt + Math.max(1, options.modelTimeoutMs), reserveMs: 0, + compaction: { + maxChars: 60_000, + keepRecent: 8, + staleTools: ["read_range", "inspect_workbook"], + }, includeRoomContext: false, systemPrompt: MANAGED_LOCK_SYSTEM_PROMPT, additionalInstructions: bridgeInstructions(task, room.artifactIds()), @@ -286,6 +338,13 @@ export async function runSpreadsheetBenchNodeAgentBridge( candidateWorkbookPath, patches: room.packageMutations(), }); + const structuralPlans = room.packageStructuralRepairs(); + const structuralRepair = structuralPlans.length > 0 + ? await (options.applyStructuralRepairs ?? applySpreadsheetBenchStructuralRepairs)({ + workbookPath: candidateWorkbookPath, + repairs: structuralPlans, + }) + : undefined; const emittedCandidateSha256 = sha256File(candidateWorkbookPath); const candidateFinalization = options.finalizeCandidate ? await options.finalizeCandidate({ @@ -332,6 +391,7 @@ export async function runSpreadsheetBenchNodeAgentBridge( artifactIds: room.artifactIds(), outcomeStatus: outcome.status, recalculation, + structuralRepair, candidateFinalization, }); @@ -354,7 +414,7 @@ export async function runSpreadsheetBenchNodeAgentBridge( candidateEmittedBeforeEvaluatorAccess: true, }, model: { - name: options.model.name, + name: effectiveModel.name, calls: frameReceipt.agentResult.usage.modelCalls, usage: { inputTokens: frameReceipt.agentResult.usage.inputTokens, @@ -365,12 +425,327 @@ export async function runSpreadsheetBenchNodeAgentBridge( }, }, recalculation, + ...(structuralRepair ? { structuralRepair } : {}), ...(candidateFinalization ? { candidateFinalization } : {}), frame: frameReceipt, trace, }; } +type BridgeDeterministicContract = { + complete: boolean; + plans: Array<{ + sheet: string; + operations: ReturnType["operations"]; + planHash: string; + }>; +}; + +/** + * A complete workbook-invariant contract should not spend hundreds of + * thousands of provider tokens asking an LLM to copy an already proven plan. + * This policy still runs through frameRunner and the production tools. It only + * handles explicitly complete contracts; all ambiguous work delegates to the + * configured model with the same message/tool history. + */ +function contractFirstBridgeModel( + delegate: AgentModel, + room: SpreadsheetBenchWorkbookRoomTools, + instruction: string, + boundedAuditPlanning: boolean, +): AgentModel { + let delegated = false; + let deterministicStarted = false; + let phase: "idle" + | "inspect_requested" + | "evidence_inspect_requested" + | "unresolved_inspect_requested" + | "execute_requested" + | "structure_execute_requested" = "idle"; + let activePlan: BridgeDeterministicContract["plans"][number] | undefined; + let activeStructuralPlan: SpreadsheetBenchStructuralRepairPlan | undefined; + let callCounter = 0; + const completedPlanHashes = new Set(); + const completedStructuralRepairIds = new Set(); + + return { + get name() { + return delegated ? delegate.name : "nodeagent/workbook-contract"; + }, + ...(delegate.routeState ? { routeState: () => delegate.routeState!() } : {}), + async next(input) { + if (delegated) return boundedBridgeDelegateNext(delegate, input); + + const latest = latestBridgeToolResult(input.messages); + if (phase === "unresolved_inspect_requested") { + return { + text: latest?.tool === "inspect_workbook" && asRecord(latest.result)?.ok === true + ? "The visible workbook inspection found no high-confidence write contract. The audit remains unresolved without mutation." + : "The bounded workbook inspection did not produce executable evidence. The audit remains unresolved without mutation.", + toolCalls: [], + done: true, + usage: { inputTokens: 0, outputTokens: 0 }, + }; + } + if (phase === "evidence_inspect_requested") { + if (latest?.tool !== "inspect_workbook" || asRecord(latest.result)?.ok !== true) { + return { + text: "The bounded workbook inspection failed, so no provider-authored mutation was attempted.", + toolCalls: [], + done: true, + usage: { inputTokens: 0, outputTokens: 0 }, + }; + } + delegated = true; + return boundedBridgeDelegateNext(delegate, input); + } + if (phase === "inspect_requested") { + if (latest?.tool !== "inspect_workbook" || asRecord(latest.result)?.ok !== true || (!activePlan && !activeStructuralPlan)) { + delegated = true; + return boundedBridgeDelegateNext(delegate, input); + } + if (activeStructuralPlan) { + phase = "structure_execute_requested"; + return { + text: `Executing the complete visible structural workbook contract for ${activeStructuralPlan.sheet}.`, + toolCalls: [{ + id: `workbook-contract-${++callCounter}`, + tool: EXECUTE_WORKBOOK_STRUCTURE_REPAIR_TOOL_NAME, + args: { + instruction, + artifactId: activeStructuralPlan.sheet, + repairId: activeStructuralPlan.repairId, + }, + }], + done: false, + usage: { inputTokens: 0, outputTokens: 0 }, + }; + } + const plan = activePlan; + if (!plan) { + delegated = true; + return boundedBridgeDelegateNext(delegate, input); + } + phase = "execute_requested"; + return { + text: `Executing the complete visible workbook contract for ${plan.sheet}.`, + toolCalls: [{ + id: `workbook-contract-${++callCounter}`, + tool: EXECUTE_VERIFIED_WORKBOOK_PLAN_TOOL_NAME, + args: { + instruction, + artifactId: plan.sheet, + maxCells: 200, + reason: "complete visible workbook invariant contract", + }, + }], + done: false, + usage: { inputTokens: 0, outputTokens: 0 }, + }; + } + + if (phase === "structure_execute_requested") { + const result = asRecord(latest?.result); + if (latest?.tool !== EXECUTE_WORKBOOK_STRUCTURE_REPAIR_TOOL_NAME + || result?.status !== "completed" + || !activeStructuralPlan) { + delegated = true; + return boundedBridgeDelegateNext(delegate, input); + } + completedStructuralRepairIds.add(activeStructuralPlan.repairId); + activeStructuralPlan = undefined; + phase = "idle"; + } + + if (phase === "execute_requested") { + const result = asRecord(latest?.result); + if (latest?.tool !== EXECUTE_VERIFIED_WORKBOOK_PLAN_TOOL_NAME || result?.status !== "completed" || !activePlan) { + delegated = true; + return boundedBridgeDelegateNext(delegate, input); + } + completedPlanHashes.add(activePlan.planHash); + activePlan = undefined; + phase = "idle"; + } + + const structuralContract = room.workbookStructureRepairContract({ instruction }); + if (structuralContract && !completedStructuralRepairIds.has(structuralContract.repairId)) { + deterministicStarted = true; + activeStructuralPlan = structuralContract; + phase = "inspect_requested"; + return { + text: `Inspecting ${structuralContract.sheet} before a governed structural workbook repair.`, + toolCalls: [{ + id: `workbook-contract-${++callCounter}`, + tool: "inspect_workbook", + args: { instruction, artifactId: structuralContract.sheet, maxCells: 200 }, + }], + done: false, + usage: { inputTokens: 0, outputTokens: 0 }, + }; + } + + const contract = bridgeDeterministicContract(room); + if (!deterministicStarted) { + if (!contract.complete || contract.plans.length === 0) { + const inspection = room.taskInspection(); + if (boundedAuditPlanning && inspection.mutatingTask && inspection.auditFocus) { + const evidenceArtifact = bridgeInspectionEvidenceArtifact(inspection) ?? room.artifactIds()[0]; + phase = bridgeInspectionHasWriteEvidence(inspection) + ? "evidence_inspect_requested" + : "unresolved_inspect_requested"; + deterministicStarted = true; + return { + text: bridgeInspectionHasWriteEvidence(inspection) + ? "Inspecting the highest-confidence visible audit evidence before bounded model planning." + : "Inspecting the workbook once before recording an evidence-bounded unresolved audit.", + toolCalls: [{ + id: `workbook-contract-${++callCounter}`, + tool: "inspect_workbook", + args: { + instruction, + ...(evidenceArtifact ? { artifactId: evidenceArtifact } : {}), + maxCells: 200, + }, + }], + done: false, + usage: { inputTokens: 0, outputTokens: 0 }, + }; + } + delegated = true; + return boundedBridgeDelegateNext(delegate, input); + } + deterministicStarted = true; + } + + const nextPlan = contract.plans.find((plan) => !completedPlanHashes.has(plan.planHash)); + if (nextPlan) { + activePlan = nextPlan; + phase = "inspect_requested"; + return { + text: `Inspecting ${nextPlan.sheet} before a bounded deterministic workbook repair.`, + toolCalls: [{ + id: `workbook-contract-${++callCounter}`, + tool: "inspect_workbook", + args: { instruction, artifactId: nextPlan.sheet, maxCells: 200 }, + }], + done: false, + usage: { inputTokens: 0, outputTokens: 0 }, + }; + } + + if (contract.plans.length > 0 || room.changedCellCount() === 0) { + delegated = true; + return boundedBridgeDelegateNext(delegate, input); + } + return { + text: "The complete visible workbook contract passed bounded preflight, managed writes, and post-write verification.", + toolCalls: [], + done: true, + usage: { inputTokens: 0, outputTokens: 0 }, + }; + }, + }; +} + +const MAX_CONSECUTIVE_FAILED_PREFLIGHTS = 3; + +async function boundedBridgeDelegateNext( + delegate: AgentModel, + input: Parameters[0], +): ReturnType { + const failedPreflights = consecutiveFailedBridgePreflights(input.messages); + if (failedPreflights >= MAX_CONSECUTIVE_FAILED_PREFLIGHTS) { + return { + text: `The bounded workbook repair budget ended after ${failedPreflights} consecutive failed preflights. No unverified mutation was applied.`, + toolCalls: [], + done: true, + usage: { inputTokens: 0, outputTokens: 0 }, + }; + } + return delegate.next(input); +} + +function consecutiveFailedBridgePreflights( + messages: Parameters[0]["messages"], +): number { + let failures = 0; + for (const message of messages) { + if (message.role !== "tool" || message.toolName !== "verify_workbook") continue; + const result = parseBridgeToolResult(message.content); + if (result?.phase !== "preflight") continue; + failures = result.status === "passed" && result.ok !== false ? 0 : failures + 1; + } + return failures; +} + +function bridgeInspectionHasWriteEvidence(inspection: WorkbookTaskInspection): boolean { + return inspection.formulaRepairSuggestions.length > 0 + || inspection.valueSuggestions.length > 0 + || inspection.styleSuggestions.length > 0 + || inspection.formulaFillSuggestions.some((suggestion) => suggestion.operations.length > 0); +} + +function bridgeInspectionNeedsBoundedNoEvidenceCompletion(inspection: WorkbookTaskInspection): boolean { + return inspection.mutatingTask + && !!inspection.auditFocus + && inspection.deterministicPlan?.status !== "complete" + && !bridgeInspectionHasWriteEvidence(inspection); +} + +function bridgeInspectionEvidenceArtifact(inspection: WorkbookTaskInspection): string | undefined { + return inspection.formulaRepairSuggestions[0]?.sheet + ?? inspection.valueSuggestions[0]?.sheet + ?? inspection.styleSuggestions[0]?.sheet + ?? inspection.formulaFillSuggestions[0]?.sheet + ?? inspection.recommendedReads[0]?.sheet + ?? inspection.referencedSheets[0]; +} + +function parseBridgeToolResult(content: string): Record | undefined { + try { + return asRecord(JSON.parse(content)); + } catch { + return undefined; + } +} + +function bridgeDeterministicContract(room: SpreadsheetBenchWorkbookRoomTools): BridgeDeterministicContract { + const inspection = room.taskInspection(); + if (inspection.deterministicPlan?.status !== "complete") return { complete: false, plans: [] }; + const plans = inspection.deterministicPlan.sheets.map((sheet) => { + const suggested = buildWorkbookSuggestedPlan(inspection, sheet); + return { + sheet, + operations: suggested.operations, + conflicts: suggested.conflicts, + planHash: stableTraceHash({ sheet, operations: suggested.operations }), + }; + }); + const operationCount = plans.reduce((total, plan) => total + plan.operations.length, 0); + const complete = plans.every((plan) => plan.conflicts.length === 0 && plan.operations.length > 0) + && operationCount === inspection.deterministicPlan.operationCount; + return { + complete, + plans: complete + ? plans.map(({ sheet, operations, planHash }) => ({ sheet, operations, planHash })) + : [], + }; +} + +function latestBridgeToolResult(messages: Parameters[0]["messages"]): { + tool: string; + result: unknown; +} | undefined { + const message = [...messages].reverse().find((candidate) => candidate.role === "tool" && candidate.toolName); + if (!message?.toolName) return undefined; + try { + return { tool: message.toolName, result: JSON.parse(message.content) as unknown }; + } catch { + return { tool: message.toolName, result: message.content }; + } +} + class SpreadsheetBenchWorkbookRoomTools implements RoomTools { private readonly workbook: ExcelJS.Workbook; private readonly instruction: string; @@ -383,13 +758,17 @@ class SpreadsheetBenchWorkbookRoomTools implements RoomTools { sheet: string; address: string; numFmtTouched: boolean; + fontColorTouched: boolean; originalState: WorkbookCellSemanticState; }>(); + private readonly structuralRepairs: SpreadsheetBenchStructuralRepairPlan[] = []; private readonly chat: string[] = []; private lockCounter = 0; private draftCounter = 0; private workbookVersion = 1; private mutations = 0; + private initialDeterministicInspection?: WorkbookTaskInspection; + private inspectionCache?: { workbookVersion: number; inspection: WorkbookTaskInspection }; constructor(args: { workbook: ExcelJS.Workbook; @@ -418,32 +797,176 @@ class SpreadsheetBenchWorkbookRoomTools implements RoomTools { .sort((left, right) => left.sheet.localeCompare(right.sheet) || left.address.localeCompare(right.address)) .flatMap((target) => { const cell = this.sheet(target.sheet).getCell(target.address); + const currentState = workbookCellSemanticState(cell); if (sameWorkbookCellSemanticState( target.originalState, - workbookCellSemanticState(cell), + currentState, target.numFmtTouched, + target.fontColorTouched, )) return []; - return [workbookCellPatch(target.sheet, target.address, cell, target.numFmtTouched)]; + return [workbookCellPatch( + target.sheet, + target.address, + cell, + target.originalState, + target.numFmtTouched, + target.fontColorTouched, + )]; }); } - taskInspection() { - this.recalculateChangedFormulas(); - return inspectWorkbookTask({ + packageStructuralRepairs(): SpreadsheetBenchStructuralRepairPlan[] { + return this.structuralRepairs.map((repair) => ({ + ...repair, + formulaRepairs: repair.formulaRepairs.map((formula) => ({ ...formula })), + evidence: [...repair.evidence], + })); + } + + workbookStructureRepairContract(args: { + instruction: string; + artifactId?: string; + }): SpreadsheetBenchStructuralRepairPlan | undefined { + if (args.instruction.trim() !== this.instruction.trim()) return undefined; + const plan = detectSpreadsheetBenchStructuralRepair({ instruction: this.instruction, sheetNames: this.artifactIds(), cells: this.observedCells(), }); + if (args.artifactId && plan?.sheet.toLowerCase() !== args.artifactId.toLowerCase()) return undefined; + return plan; } - async snapshot(artifactId?: string): Promise { + executeWorkbookStructureRepair(args: { + instruction: string; + artifactId?: string; + repairId: string; + }): Record { + const plan = this.workbookStructureRepairContract(args); + if (!plan || plan.repairId !== args.repairId) { + return { + ok: false, + status: "needs_repair", + operationCount: 0, + phases: { + preflight: { status: "needs_repair", issues: ["stale_or_missing_structural_contract"] }, + write: { status: "skipped" }, + verify: { status: "skipped" }, + }, + }; + } + + const worksheet = this.sheet(plan.sheet); + worksheet.spliceRows(plan.insertRow, 0, []); + worksheet.getCell(plan.labelCell).value = plan.label; + worksheet.getCell(plan.selectorCell).value = plan.selectorValue; + let formulaReplacementCount = 0; + const formulaReplacementTargets: string[] = []; + worksheet.eachRow({ includeEmpty: false }, (row) => { + row.eachCell({ includeEmpty: false }, (cell) => { + const formula = cellFormula(cell); + if (!formula || !formula.includes(plan.formulaSearch)) return; + cell.value = { + formula: formula.replaceAll(plan.formulaSearch, plan.formulaReplace), + result: 0, + } as ExcelJS.CellValue; + formulaReplacementCount += 1; + formulaReplacementTargets.push(`${worksheet.name}!${cell.address}`); + }); + }); + if (formulaReplacementCount !== plan.expectedFormulaReplacementCount) { + throw new Error( + `Structural repair ${plan.repairId} expected ${plan.expectedFormulaReplacementCount} formula replacements but applied ${formulaReplacementCount}`, + ); + } + for (const formulaRepair of plan.formulaRepairs) { + this.sheet(formulaRepair.sheet).getCell(formulaRepair.cell).value = { + formula: formulaRepair.formula.replace(/^=/, ""), + result: 0, + } as ExcelJS.CellValue; + } + this.structuralRepairs.push(plan); + this.workbookVersion += 1; + this.mutations += plan.operationCount; + const targets = [ + `${worksheet.name}!${plan.insertRow}:${plan.insertRow}`, + `${worksheet.name}!${plan.labelCell}`, + `${worksheet.name}!${plan.selectorCell}`, + ...formulaReplacementTargets, + ...plan.formulaRepairs.map((repair) => `${repair.sheet}!${repair.cell}`), + ]; + if (targets.length !== plan.operationCount) { + throw new Error( + `Structural repair ${plan.repairId} expected ${plan.operationCount} mutation targets but recorded ${targets.length}`, + ); + } + + const remaining = detectSpreadsheetBenchStructuralRepair({ + instruction: this.instruction, + sheetNames: this.artifactIds(), + cells: this.observedCells(), + }); + const verified = !remaining; + return { + ok: verified, + status: verified ? "completed" : "needs_repair", + repairId: plan.repairId, + operationCount: plan.operationCount, + targets, + formulaReplacementCount, + explicitFormulaRepairCount: plan.formulaRepairs.length, + phases: { + preflight: { + status: "passed", + basis: plan.basis, + evidence: plan.evidence, + }, + write: { + status: "completed", + insertedRowCount: 1, + formulaReplacementCount, + explicitFormulaRepairCount: plan.formulaRepairs.length, + }, + verify: { + status: verified ? "passed" : "needs_repair", + remainingRepairId: remaining?.repairId, + }, + }, + }; + } + + taskInspection() { this.recalculateChangedFormulas(); - const allCells = this.observedCells(); - const inspection = inspectWorkbookTask({ + if (this.inspectionCache?.workbookVersion === this.workbookVersion) { + return this.inspectionCache.inspection; + } + const cells = this.observedCells(); + const current = inspectWorkbookTask({ instruction: this.instruction, sheetNames: this.artifactIds(), - cells: allCells, + cells, }); + if (!this.initialDeterministicInspection && current.deterministicPlan?.status === "complete") { + this.initialDeterministicInspection = current; + } + const inspection = this.initialDeterministicInspection + ? remainingDeterministicInspection(this.initialDeterministicInspection, current, cells) + : current; + this.inspectionCache = { workbookVersion: this.workbookVersion, inspection }; + return inspection; + } + + verifiedWorkbookInspection(args: { instruction: string; artifactId: string }): WorkbookTaskInspection | undefined { + if (args.instruction.trim() !== this.instruction.trim()) return undefined; + const inspection = this.taskInspection(); + if (inspection.deterministicPlan?.status !== "complete") return undefined; + return focusDeterministicInspection(inspection, args.artifactId); + } + + async snapshot(artifactId?: string): Promise { + this.recalculateChangedFormulas(); + const allCells = this.observedCells(); + const inspection = this.taskInspection(); const sheet = artifactId ? this.sheet(artifactId) : this.preferredInspectionSheet(allCells, inspection); const selected = selectWorkbookTaskCells({ inspection, @@ -620,6 +1143,7 @@ class SpreadsheetBenchWorkbookRoomTools implements RoomTools { sheet: target.sheet.name, address: target.address, numFmtTouched: priorTarget?.numFmtTouched === true || typeof asRecord(value)?.numFmt === "string", + fontColorTouched: priorTarget?.fontColorTouched === true || normalizeSpreadsheetFontColor(asRecord(value)?.fontColor) !== undefined, originalState: priorTarget?.originalState ?? originalState, }); this.workbookVersion += 1; @@ -672,6 +1196,11 @@ class SpreadsheetBenchWorkbookRoomTools implements RoomTools { left.sheet.localeCompare(right.sheet) || left.address.localeCompare(right.address))) { const sheet = this.sheet(target.sheet); const cell = sheet.getCell(target.address); + const currentState = workbookCellSemanticState(cell); + const contentUnchanged = target.originalState.kind === currentState.kind + && target.originalState.formula === currentState.formula + && target.originalState.valueKey === currentState.valueKey; + if (contentUnchanged) continue; const formula = cellFormula(cell); if (!formula) continue; attemptedFormulaCount += 1; @@ -780,6 +1309,9 @@ class SpreadsheetBenchWorkbookRoomTools implements RoomTools { for (const suggestion of inspection.formulaRepairSuggestions) { scores.set(suggestion.sheet, (scores.get(suggestion.sheet) ?? 0) + (suggestion.confidence === "high" ? 24 : 8)); } + for (const suggestion of inspection.styleSuggestions) { + scores.set(suggestion.sheet, (scores.get(suggestion.sheet) ?? 0) + 24); + } const selected = [...scores.entries()] .sort((left, right) => right[1] - left[1] || this.artifactIds().indexOf(left[0]) - this.artifactIds().indexOf(right[0]))[0]; return selected && selected[1] > 0 ? this.sheet(selected[0]) : this.workbook.worksheets[0]; @@ -907,10 +1439,15 @@ function assertCandidateDoesNotOverwriteAgentInput(candidatePath: string, task: } } -function buildBridgeFrame(manifest: StagedAgentManifest, artifactIds: string[], traceId: string): ReasoningFrame { +function buildBridgeFrame( + manifest: StagedAgentManifest, + artifactIds: string[], + traceId: string, + goalOverride?: string, +): ReasoningFrame { return { frameId: `rf_${traceId.replace(/[^a-z0-9_-]/gi, "_").slice(0, 100)}`, - goal: manifest.instruction, + goal: goalOverride ?? manifest.instruction, phase: "execute", status: "pending", contextPack: { @@ -944,10 +1481,11 @@ function bridgeInstructions(task: OpenedAgentTask, artifactIds: string[]): strin `Agent-visible input workbook name: ${basename(task.sourceWorkbookPath)}. Use descriptive filename terms as audit clues, then confirm them against workbook evidence.`, `Visible worksheet artifact IDs: ${artifactIds.map((id) => JSON.stringify(id)).join(", ")}.`, `Complete task instruction: ${task.manifest.instruction}`, - "Treat high-confidence target bands and value/formula suggestions from inspect_workbook as the target-selection contract. Do not replace them with unlabeled blank cells, and preflight every high-confidence target before writing.", + "Treat the agent-visible filename only as an audit-class hypothesis. Confirm every edit against local workbook evidence, make the smallest verified change, and do not treat unrelated blanks or inferred anomalies as mandatory targets.", + "Treat explicit target bands and locally confirmed value/formula/style suggestions from inspect_workbook as the target-selection contract. Preserve content during style-only writes, and preflight every proposed target before writing.", "The first verify_workbook call for a proposed operation set is the durable plan/preflight boundary. Do not write a plan that returns needs_repair; submit a corrected replacement plan first.", "When inspect_workbook returns a complete high-confidence formula/value contract, prefer execute_verified_workbook_plan with the same task instruction and artifactId. It performs deterministic plan materialization, preflight, managed-lock writes, and post-write verification without requiring a large echoed operation array.", - "For formula writes, pass write_locked_cell(s) direct formula, result, and optional numFmt fields; pass the same formula/result/numFmt operations to verify_workbook.", + "For writes, pass write_locked_cell(s) direct formula/result, value, numFmt, and/or fontColor fields; pass the same touched fields to verify_workbook. A fontColor-only operation must preserve the existing value or formula.", "When inspect_workbook returns workbookWideRepairContract, complete every listed repair one worksheet at a time: inspect that artifact, preflight its complete repair set, write it, and post-verify it before moving to the next artifact.", "After every managed write, call verify_workbook with afterWrite=true for all changed targets. Use repairPrompt for at most one focused repair before reporting unresolved work.", "No evaluator manifest, gold workbook, answer position, or scorer metadata exists in this frame or in any available tool.", @@ -1224,6 +1762,112 @@ type BridgeWorkbookRepair = { evidence: string[]; }; +function remainingDeterministicInspection( + initial: WorkbookTaskInspection, + current: WorkbookTaskInspection, + cells: WorkbookObservedCell[], +): WorkbookTaskInspection { + const initialContract = initial.deterministicPlan; + if (!initialContract) return current; + const cellsByKey = new Map(cells.map((cell) => [workbookCellKey(cell.sheet, cell.address), cell])); + const remainingPlans = initialContract.sheets.flatMap((sheet) => { + const suggested = buildWorkbookSuggestedPlan(initial, sheet); + if (suggested.conflicts.length > 0) return []; + const operations = suggested.operations.filter((operation) => !bridgeSuggestedOperationSatisfied( + operation, + cellsByKey.get(workbookCellKey(sheet, operation.elementId)), + )); + return operations.length > 0 ? [{ sheet, operations }] : []; + }); + const remainingKeys = new Set(remainingPlans.flatMap((plan) => + plan.operations.map((operation) => workbookCellKey(plan.sheet, operation.elementId)))); + const { deterministicPlan: _currentPlan, ...currentWithoutDeterministicPlan } = current; + if (remainingKeys.size === 0) { + return { + ...currentWithoutDeterministicPlan, + formulaFillSuggestions: [], + formulaRepairSuggestions: [], + valueSuggestions: [], + styleSuggestions: [], + }; + } + const keep = (sheet: string, cell: string) => remainingKeys.has(workbookCellKey(sheet, cell)); + return { + ...currentWithoutDeterministicPlan, + ...(initial.auditFocus ? { auditFocus: initial.auditFocus } : {}), + deterministicPlan: { + ...initialContract, + operationCount: remainingKeys.size, + sheets: remainingPlans.map((plan) => plan.sheet), + }, + targetCandidates: initial.targetCandidates.filter((target) => keep(target.sheet, target.address)), + blockedTargets: [], + findings: initial.findings.filter((finding) => keep(finding.sheet, finding.address)), + formulaFillSuggestions: initial.formulaFillSuggestions.flatMap((suggestion) => { + const operations = suggestion.operations.filter((operation) => keep(operation.sheet, operation.cell)); + return operations.length > 0 ? [{ ...suggestion, operations }] : []; + }), + formulaRepairSuggestions: initial.formulaRepairSuggestions.filter((suggestion) => keep(suggestion.sheet, suggestion.cell)), + valueSuggestions: initial.valueSuggestions.filter((suggestion) => keep(suggestion.sheet, suggestion.cell)), + styleSuggestions: initial.styleSuggestions.filter((suggestion) => keep(suggestion.sheet, suggestion.cell)), + rankedCellKeys: initial.rankedCellKeys.filter((key) => remainingKeys.has(key)), + recommendedReads: remainingPlans.map((plan) => ({ + sheet: plan.sheet, + addresses: plan.operations.map((operation) => operation.elementId), + reason: "durable complete visible-workbook contract", + })), + }; +} + +function focusDeterministicInspection( + inspection: WorkbookTaskInspection, + artifactId: string, +): WorkbookTaskInspection | undefined { + const sheet = inspection.deterministicPlan?.sheets.find((candidate) => candidate.toLowerCase() === artifactId.toLowerCase()); + if (!sheet || !inspection.deterministicPlan) return undefined; + const keepSheet = (candidate: string) => candidate.toLowerCase() === sheet.toLowerCase(); + const suggested = buildWorkbookSuggestedPlan(inspection, sheet); + if (suggested.operations.length === 0 || suggested.conflicts.length > 0) return undefined; + return { + ...inspection, + deterministicPlan: { + ...inspection.deterministicPlan, + operationCount: suggested.operations.length, + sheets: [sheet], + }, + referencedSheets: [sheet], + targetCandidates: inspection.targetCandidates.filter((target) => keepSheet(target.sheet)), + blockedTargets: inspection.blockedTargets.filter((target) => keepSheet(target.sheet)), + targetBands: inspection.targetBands.filter((band) => keepSheet(band.sheet)), + dependencyCandidates: inspection.dependencyCandidates.filter((target) => keepSheet(target.sheet)), + findings: inspection.findings.filter((finding) => keepSheet(finding.sheet)), + formulaFillSuggestions: inspection.formulaFillSuggestions.flatMap((suggestion) => { + if (!keepSheet(suggestion.sheet)) return []; + const operations = suggestion.operations.filter((operation) => keepSheet(operation.sheet)); + return operations.length > 0 ? [{ ...suggestion, operations }] : []; + }), + formulaRepairSuggestions: inspection.formulaRepairSuggestions.filter((suggestion) => keepSheet(suggestion.sheet)), + valueSuggestions: inspection.valueSuggestions.filter((suggestion) => keepSheet(suggestion.sheet)), + styleSuggestions: inspection.styleSuggestions.filter((suggestion) => keepSheet(suggestion.sheet)), + rankedCellKeys: inspection.rankedCellKeys.filter((key) => key.startsWith(`${sheet.toLowerCase()}!`)), + recommendedReads: inspection.recommendedReads.filter((read) => keepSheet(read.sheet)), + }; +} + +function bridgeSuggestedOperationSatisfied( + operation: ReturnType["operations"][number], + cell: WorkbookObservedCell | undefined, +): boolean { + if (!cell) return false; + if (operation.formula && normalizeFormula(cell.formula) !== normalizeFormula(operation.formula)) return false; + if (Object.prototype.hasOwnProperty.call(operation, "value") + && stableTraceHash(cell.value) !== stableTraceHash(operation.value)) return false; + if (operation.numFmt && cell.numFmt !== operation.numFmt) return false; + if (operation.fontColor + && normalizeSpreadsheetFontColor(cell.fontColor) !== normalizeSpreadsheetFontColor(operation.fontColor)) return false; + return true; +} + function bridgeWorkbookInspection(rt: RoomTools): WorkbookTaskInspection | undefined { const provider = rt as RoomTools & { taskInspection?: () => WorkbookTaskInspection }; return typeof provider.taskInspection === "function" ? provider.taskInspection() : undefined; @@ -1232,6 +1876,7 @@ function bridgeWorkbookInspection(rt: RoomTools): WorkbookTaskInspection | undef function bridgeWorkbookRepairContract(rt: RoomTools): { schema: 1; requiredRepairs: BridgeWorkbookRepair[] } { const inspection = bridgeWorkbookInspection(rt); if (!inspection) return { schema: 1, requiredRepairs: [] }; + if (inspection.auditFocus) return { schema: 1, requiredRepairs: [] }; const rangeTargets = new Set(inspection.findings .filter((finding) => finding.kind === "formula_range_anomaly") .map((finding) => `${finding.sheet.toLowerCase()}!${normalizeAddress(finding.address)}`)); @@ -1386,6 +2031,7 @@ type NormalizedBridgeOperation = { formula?: string; value?: unknown; numFmt?: string; + fontColor?: string; }; function normalizedBridgeOperations(args: unknown, source: "verify" | "write", defaultArtifactId = ""): NormalizedBridgeOperation[] { @@ -1418,6 +2064,7 @@ function normalizedBridgeOperations(args: unknown, source: "verify" | "write", d const numFmtValue = typeof operation.numFmt === "string" ? operation.numFmt : typeof nested?.numFmt === "string" ? nested.numFmt : undefined; + const fontColor = normalizeSpreadsheetFontColor(operation.fontColor ?? nested?.fontColor); const parsedTarget = bridgeElementTarget(operation.elementId, artifactId); const target = `${parsedTarget.sheet.toLowerCase()}!${parsedTarget.address}`; return [{ @@ -1425,6 +2072,7 @@ function normalizedBridgeOperations(args: unknown, source: "verify" | "write", d ...(formula ? { formula } : {}), ...(!formula && (hasResult || hasValue) ? { value } : {}), ...(numFmtValue?.trim() ? { numFmt: numFmtValue.trim() } : {}), + ...(fontColor ? { fontColor } : {}), }]; }).sort((left, right) => left.target.localeCompare(right.target)); } @@ -1445,8 +2093,11 @@ function bridgePlanState( function bridgeOperationArgs(operation: NormalizedBridgeOperation): Record { return { elementId: operation.target.slice(operation.target.lastIndexOf("!") + 1), - ...(operation.formula ? { formula: operation.formula } : { value: operation.value }), + ...(operation.formula + ? { formula: operation.formula } + : Object.prototype.hasOwnProperty.call(operation, "value") ? { value: operation.value } : {}), ...(operation.numFmt ? { numFmt: operation.numFmt } : {}), + ...(operation.fontColor ? { fontColor: operation.fontColor } : {}), }; } @@ -1550,6 +2201,7 @@ function focusBridgeInspectionResult( evidence: repair.evidence, })), valueSuggestions: [], + styleSuggestions: [], rankedCellKeys: repairs.map((repair) => `${repair.sheet.toLowerCase()}!${normalizeAddress(repair.cell)}`), recommendedReads: [{ sheet: artifactId, @@ -1581,7 +2233,7 @@ function buildStageReceipts( ): Record { const indexed = frameReceipt.agentResult.trace.map((event, eventIndex) => ({ event, eventIndex })); const inspect = indexed.filter(({ event }) => event.tool === "inspect_workbook"); - const composite = indexed.filter(({ event }) => event.tool === EXECUTE_VERIFIED_WORKBOOK_PLAN_TOOL_NAME); + const composite = indexed.filter(({ event }) => COMPOSITE_WORKBOOK_TOOL_NAMES.has(event.tool)); const verifications = indexed.filter(({ event }) => event.tool === "verify_workbook"); const explicitPreflights = verifications.filter(({ event }) => verificationPhase(event.args, event.result) === "preflight"); const explicitPostWrites = verifications.filter(({ event }) => verificationPhase(event.args, event.result) === "post_write"); @@ -1754,6 +2406,7 @@ function buildBridgeTrace(args: { artifactIds: string[]; outcomeStatus: SpreadsheetBenchNodeAgentBridgeReceipt["outcome"]["status"]; recalculation: SpreadsheetBenchNodeAgentRecalculationReceipt; + structuralRepair?: SpreadsheetBenchStructuralRepairReceipt; candidateFinalization?: SpreadsheetBenchCandidateFinalizationReceipt; }): NodeAgentTrace { const candidateRef = traceRef("artifact", args.candidateWorkbookPath, { @@ -1807,6 +2460,20 @@ function buildBridgeTrace(args: { verifier: args.recalculation.engine, status: args.recalculation.unresolvedFormulaCount === 0 ? "verified" : "needs_review", })); + if (args.structuralRepair) { + trace.evidence.push(makeEvidenceReceipt({ + traceId: args.traceId, + label: "SpreadsheetBench structural workbook repair", + sourceRefs: [traceRef("tool_result", `${args.traceId}:structural-repair`, { + label: args.structuralRepair.backend, + hash: stableTraceHash(args.structuralRepair), + })], + artifactRefs: [candidateRef], + fact: args.structuralRepair, + verifier: args.structuralRepair.backend, + status: "verified", + })); + } if (args.candidateFinalization) { trace.evidence.push(makeEvidenceReceipt({ traceId: args.traceId, @@ -1844,8 +2511,17 @@ function eventTraceRef(event: SpreadsheetBenchNodeAgentBridgeEventRef): TraceRef function mutationStatus(result: unknown): MutationReceipt["status"] { const record = asRecord(result); if (record?.pendingApproval === true) return "pending_approval"; + const resultItems = Array.isArray(record?.results) + ? record.results.map(asRecord).filter((item): item is Record => Boolean(item)) + : []; + const hasCommittedResult = resultItems.some((item) => item.ok === true && item.skipped !== true); + if (hasCommittedResult || (typeof record?.changedTargetCount === "number" && record.changedTargetCount > 0)) { + return "committed"; + } + if (record?.alreadySatisfied === true) return "skipped"; if (record?.conflict === true) return "conflict"; if (record?.skipped === true) return "skipped"; + if (resultItems.length > 0 && resultItems.every((item) => item.skipped === true)) return "skipped"; if (record?.ok === true) return "committed"; return "conflict"; } @@ -1861,11 +2537,17 @@ function writeTargets(args: unknown, result?: unknown): string[] { }); if (typeof record.elementId === "string") targets.push(record.elementId); const resultRecord = asRecord(result); + if (Array.isArray(resultRecord?.targets)) { + for (const target of resultRecord.targets) { + if (typeof target === "string") targets.push(target); + } + } const plan = asRecord(asRecord(resultRecord?.phases)?.plan); if (Array.isArray(plan?.targets)) { for (const target of plan.targets) if (typeof target === "string") targets.push(target); } - return [...new Set(targets.map((target) => artifactId ? `${artifactId}!${target}` : target))]; + return [...new Set(targets.map((target) => + artifactId && !target.includes("!") ? `${artifactId}!${target}` : target))]; } function verificationPhase(args: unknown, result: unknown): "preflight" | "post_write" { @@ -1897,7 +2579,7 @@ function bridgeEventVerificationStatus( phase: "preflight" | "verify", ): "passed" | "needs_repair" | "missing" { if (!event) return "missing"; - return event.tool === EXECUTE_VERIFIED_WORKBOOK_PLAN_TOOL_NAME + return COMPOSITE_WORKBOOK_TOOL_NAMES.has(event.tool) ? compositePhaseStatus(event.result, phase) : verificationStatus(event.result); } @@ -1924,12 +2606,14 @@ function operationCount(args: unknown): number { } function observedCell(sheet: ExcelJS.Worksheet, cell: ExcelJS.Cell, version: number): WorkbookObservedCell { + const fontColor = cellFontColor(cell); return { sheet: sheet.name, address: normalizeAddress(cell.address), value: roomCellScalar(cell), ...(cellFormula(cell) ? { formula: cellFormula(cell) } : {}), ...(cell.numFmt ? { numFmt: cell.numFmt } : {}), + ...(fontColor ? { fontColor } : {}), version, }; } @@ -1937,11 +2621,13 @@ function observedCell(sheet: ExcelJS.Worksheet, cell: ExcelJS.Cell, version: num function roomCellValue(cell: ExcelJS.Cell): unknown { const formula = cellFormula(cell); const numFmt = cell.numFmt && cell.numFmt !== "General" ? cell.numFmt : undefined; - if (formula || numFmt) { + const fontColor = cellFontColor(cell); + if (formula || numFmt || fontColor) { return { value: roomCellScalar(cell), ...(formula ? { formula } : {}), ...(numFmt ? { numFmt } : {}), + ...(fontColor ? { fontColor } : {}), }; } return cell.value; @@ -1959,24 +2645,28 @@ type WorkbookCellSemanticState = { formula?: string; valueKey?: string; numFmt: string; + fontColor?: string; }; function workbookCellSemanticState(cell: ExcelJS.Cell): WorkbookCellSemanticState { const formula = cellFormula(cell); - if (formula) return { kind: "formula", formula, numFmt: cell.numFmt }; - if (cell.value === null || cell.value === undefined) return { kind: "clear", numFmt: cell.numFmt }; - return { kind: "value", valueKey: stableWorkbookValue(cell.value), numFmt: cell.numFmt }; + const fontColor = cellFontColor(cell); + if (formula) return { kind: "formula", formula, numFmt: cell.numFmt, ...(fontColor ? { fontColor } : {}) }; + if (cell.value === null || cell.value === undefined) return { kind: "clear", numFmt: cell.numFmt, ...(fontColor ? { fontColor } : {}) }; + return { kind: "value", valueKey: stableWorkbookValue(cell.value), numFmt: cell.numFmt, ...(fontColor ? { fontColor } : {}) }; } function sameWorkbookCellSemanticState( left: WorkbookCellSemanticState, right: WorkbookCellSemanticState, compareNumberFormat: boolean, + compareFontColor: boolean, ): boolean { return left.kind === right.kind && left.formula === right.formula && left.valueKey === right.valueKey - && (!compareNumberFormat || left.numFmt === right.numFmt); + && (!compareNumberFormat || left.numFmt === right.numFmt) + && (!compareFontColor || left.fontColor === right.fontColor); } function stableWorkbookValue(value: unknown): string { @@ -1997,11 +2687,27 @@ function workbookCellPatch( sheet: string, address: string, cell: ExcelJS.Cell, + originalState: WorkbookCellSemanticState, numFmtTouched: boolean, + fontColorTouched: boolean, ): SpreadsheetBenchWorkbookCellPatch { const formula = cellFormula(cell); const valueRecord = asRecord(cell.value); const numFmt = numFmtTouched ? cell.numFmt : undefined; + const fontColorStyle = fontColorTouched ? cellFontColorStyle(cell) : {}; + const currentState = workbookCellSemanticState(cell); + const contentUnchanged = originalState.kind === currentState.kind + && originalState.formula === currentState.formula + && originalState.valueKey === currentState.valueKey; + if (contentUnchanged && (numFmtTouched || fontColorTouched)) { + return { + sheet, + address, + kind: "style", + ...(numFmt === undefined ? {} : { numFmt }), + ...fontColorStyle, + }; + } if (formula) { const hasCachedResult = Boolean(valueRecord && Object.prototype.hasOwnProperty.call(valueRecord, "result")); return { @@ -2012,6 +2718,7 @@ function workbookCellPatch( hasCachedResult, ...(hasCachedResult ? { cachedResult: valueRecord?.result } : {}), ...(numFmt === undefined ? {} : { numFmt }), + ...fontColorStyle, }; } if (cell.value === null || cell.value === undefined) { @@ -2020,6 +2727,7 @@ function workbookCellPatch( address, kind: "clear", ...(numFmt === undefined ? {} : { numFmt }), + ...fontColorStyle, }; } return { @@ -2028,6 +2736,7 @@ function workbookCellPatch( kind: "value", value: cell.value, ...(numFmt === undefined ? {} : { numFmt }), + ...fontColorStyle, }; } @@ -2074,6 +2783,10 @@ function applyRoomValue(cell: ExcelJS.Cell, value: unknown): void { : typeof value === "string" && value.trim().startsWith("=") ? value.trim().slice(1) : undefined; + const styleOnly = !!record + && !formula + && !Object.prototype.hasOwnProperty.call(record, "value") + && (typeof record.numFmt === "string" || normalizeSpreadsheetFontColor(record.fontColor) !== undefined); if (formula) { const result = record && Object.prototype.hasOwnProperty.call(record, "result") ? record.result @@ -2081,10 +2794,68 @@ function applyRoomValue(cell: ExcelJS.Cell, value: unknown): void { cell.value = { formula, ...(result === undefined ? {} : { result }) } as ExcelJS.CellValue; } else if (record && Object.prototype.hasOwnProperty.call(record, "value")) { cell.value = (record.value ?? null) as ExcelJS.CellValue; - } else { + } else if (!styleOnly) { cell.value = (value ?? null) as ExcelJS.CellValue; } + if (typeof record?.numFmt === "string" || normalizeSpreadsheetFontColor(record?.fontColor)) { + // ExcelJS reuses style objects across cells loaded from the same xf. Its + // font/numFmt setters mutate that object in place, which can silently alter + // untouched peers. Detach the complete style before changing one cell. + cell.style = cloneWorkbookCellStyle(cell.style); + } if (typeof record?.numFmt === "string") cell.numFmt = record.numFmt; + const fontColor = normalizeSpreadsheetFontColor(record?.fontColor); + if (fontColor) { + const workbook = cell.worksheet.workbook as ExcelJS.Workbook & { _themes?: Record }; + const theme = spreadsheetThemeIndexForColor(fontColor, workbook._themes?.theme1); + cell.font = { + ...cell.font, + color: theme === undefined ? { argb: fontColor } : { theme }, + }; + } +} + +function cloneWorkbookCellStyle(style: Partial): Partial { + return { + ...style, + ...(style.font ? { font: cloneWorkbookStyleValue(style.font) } : {}), + ...(style.alignment ? { alignment: cloneWorkbookStyleValue(style.alignment) } : {}), + ...(style.border ? { border: cloneWorkbookStyleValue(style.border) } : {}), + ...(style.fill ? { fill: cloneWorkbookStyleValue(style.fill) } : {}), + ...(style.protection ? { protection: cloneWorkbookStyleValue(style.protection) } : {}), + }; +} + +function cloneWorkbookStyleValue(value: T): T { + return JSON.parse(JSON.stringify(value)) as T; +} + +function cellFontColor(cell: ExcelJS.Cell): string | undefined { + const workbook = cell.worksheet.workbook as ExcelJS.Workbook & { + _themes?: Record; + }; + return resolveSpreadsheetFontColor( + cell.font?.color as SpreadsheetFontColorSource | undefined, + workbook._themes?.theme1, + ); +} + +function cellFontColorStyle(cell: ExcelJS.Cell): Pick< + SpreadsheetBenchWorkbookCellPatch, + "fontColor" | "fontColorTheme" | "fontColorTint" +> { + const fontColor = cellFontColor(cell); + if (!fontColor) return {}; + const source = cell.font?.color as SpreadsheetFontColorSource | undefined; + const theme = typeof source?.theme === "number" && Number.isInteger(source.theme) && source.theme >= 0 + ? source.theme + : undefined; + const tint = typeof source?.tint === "number" && Number.isFinite(source.tint) ? source.tint : undefined; + return { + fontColor, + ...(theme === undefined ? {} : { fontColorTheme: theme }), + ...(theme === undefined || tint === undefined ? {} : { fontColorTint: tint }), + }; } function parseReadReference(value: string): { diff --git a/src/eval/spreadsheetBenchOfficialV2Acceptance.ts b/src/eval/spreadsheetBenchOfficialV2Acceptance.ts new file mode 100644 index 00000000..e3fd1d3b --- /dev/null +++ b/src/eval/spreadsheetBenchOfficialV2Acceptance.ts @@ -0,0 +1,14 @@ +const ACCEPTED_REFRESH_STATUSES = new Set([ + "completed", + "completed_stable_pending", +] as const); + +export type SpreadsheetBenchOfficialV2AcceptedRefreshStatus = + | "completed" + | "completed_stable_pending"; + +export function isSpreadsheetBenchOfficialV2AcceptedRefreshStatus( + status: string | undefined, +): status is SpreadsheetBenchOfficialV2AcceptedRefreshStatus { + return ACCEPTED_REFRESH_STATUSES.has(status as SpreadsheetBenchOfficialV2AcceptedRefreshStatus); +} diff --git a/src/eval/spreadsheetBenchReportRepair.ts b/src/eval/spreadsheetBenchReportRepair.ts index bad22a2e..0511161c 100644 --- a/src/eval/spreadsheetBenchReportRepair.ts +++ b/src/eval/spreadsheetBenchReportRepair.ts @@ -71,6 +71,13 @@ export function mergeSpreadsheetBenchRepairReport(args: { const passCount = results.filter((result) => result.score?.pass).length; const casePassCount = caseRuns.filter((caseRun) => caseRun.pass).length; const usage = aggregateUsage(results); + const replacedWarningMarkers = replacementTaskIds.flatMap((taskId) => { + const result = baseResults.get(taskId); + return [taskId, result?.taskDir].filter((value): value is string => Boolean(value)); + }); + const baseWarnings = args.base.warnings.filter( + (warning) => !replacedWarningMarkers.some((marker) => warning.includes(marker)), + ); const { chunked: _chunked, chunkSize: _chunkSize, chunks: _chunks, ...base } = args.base as SpreadsheetBenchRunnerReport & { chunked?: boolean; chunkSize?: number; @@ -113,7 +120,7 @@ export function mergeSpreadsheetBenchRepairReport(args: { providerCostUsd: usage.costUsd, }, }, - warnings: [...new Set([...args.base.warnings, ...args.repair.warnings])], + warnings: [...new Set([...baseWarnings, ...args.repair.warnings])], caseRuns, results, repairMerge: { diff --git a/src/eval/spreadsheetBenchRunSourceFingerprint.ts b/src/eval/spreadsheetBenchRunSourceFingerprint.ts new file mode 100644 index 00000000..a8318ee3 --- /dev/null +++ b/src/eval/spreadsheetBenchRunSourceFingerprint.ts @@ -0,0 +1,59 @@ +import { createHash } from "node:crypto"; +import { existsSync, readdirSync, readFileSync } from "node:fs"; +import { relative, resolve } from "node:path"; + +const SOURCE_DIRECTORIES = ["src", "scripts", "convex"] as const; +const SOURCE_FILES = [ + "package.json", + "package-lock.json", + "tsconfig.json", + "tsconfig.app.json", + "tsconfig.node.json", + ".env", + ".env.local", + ".env.production.local", +] as const; +const IGNORED_DIRECTORIES = new Set([".git", ".tmp", "__pycache__", "node_modules"]); +const IGNORED_FILE_SUFFIXES = [".pyc", ".pyo"]; + +export interface SpreadsheetBenchRunSourceFingerprint { + sha256: string; + fileCount: number; +} + +export function fingerprintSpreadsheetBenchRunSource( + workspaceRoot = process.cwd(), +): SpreadsheetBenchRunSourceFingerprint { + const root = resolve(workspaceRoot); + const files = [ + ...SOURCE_DIRECTORIES.flatMap((directory) => listFiles(resolve(root, directory))), + ...SOURCE_FILES.map((file) => resolve(root, file)).filter(existsSync), + ].sort((left, right) => left.localeCompare(right)); + const hash = createHash("sha256"); + + for (const path of files) { + const normalized = relative(root, path).replace(/\\/g, "/"); + const content = readFileSync(path); + hash.update(`${normalized}\0${content.length}\0`, "utf8"); + hash.update(content); + hash.update("\0", "utf8"); + } + + return { sha256: hash.digest("hex"), fileCount: files.length }; +} + +function listFiles(root: string): string[] { + if (!existsSync(root)) return []; + const files: string[] = []; + for (const entry of readdirSync(root, { withFileTypes: true }).sort((left, right) => left.name.localeCompare(right.name))) { + if (entry.isDirectory() && IGNORED_DIRECTORIES.has(entry.name)) continue; + const path = resolve(root, entry.name); + if (entry.isDirectory()) { + files.push(...listFiles(path)); + continue; + } + if (!entry.isFile() || IGNORED_FILE_SUFFIXES.some((suffix) => entry.name.endsWith(suffix))) continue; + files.push(path); + } + return files; +} diff --git a/src/eval/spreadsheetBenchRunner.ts b/src/eval/spreadsheetBenchRunner.ts index b707ec99..bf4216cc 100644 --- a/src/eval/spreadsheetBenchRunner.ts +++ b/src/eval/spreadsheetBenchRunner.ts @@ -35,7 +35,56 @@ import { export type SpreadsheetBenchRunnerMode = "copy-input-baseline" | "apply-agent-patch" | "model-edit-plan" | "nodeagent-workbook"; +export type SpreadsheetBenchExcelFinalizationStatus = + | "completed" + | "completed_stable_pending" + | "not_required" + | "preserved_pending" + | "preserved_unsupported" + | "preserved_error"; + +export type SpreadsheetBenchExcelFormulaTopology = { + safe: boolean; + supportedTypes?: string[]; + counts?: Record; + cacheTargetCellCount?: number; + unsupported?: Array>; +}; + +export type SpreadsheetBenchExcelFinalizationReceipt = SpreadsheetBenchCandidateFinalizationReceipt & { + status: SpreadsheetBenchExcelFinalizationStatus; + reason: string; + calculationStates?: { + beforeCacheRead: number; + afterCacheRead: number; + }; + formulaTopology?: SpreadsheetBenchExcelFormulaTopology; + calculationStability?: { + mode: "stable_pending_opt_in"; + passed: true; + requiredIdenticalReads: number; + observedIdenticalReads: number; + observedReads: number; + sampleIntervalMs: number; + timeoutMs: number; + }; + formulaTopologyPreservation?: { + matched: true; + beforeSha256: string; + afterSha256: string; + formulaElementCount: number; + }; +}; + const FORMULA_RESULT_POLICY = "deterministic_local_subset"; +const EXCEL_FINALIZATION_STATUSES = new Set([ + "completed", + "completed_stable_pending", + "not_required", + "preserved_pending", + "preserved_unsupported", + "preserved_error", +]); const SUPPORTED_FORMULA_FUNCTIONS = [ "SUM", "AVERAGE", @@ -90,6 +139,7 @@ export type SpreadsheetBenchRunnerOptions = { modelSnapshotMaxCellChars?: number; modelRepairAttempts?: number; refreshExcelCaches?: boolean; + allowStablePendingCaches?: boolean; taskIds?: string[]; repeats?: number; retryFailed?: number; @@ -233,6 +283,7 @@ export type SpreadsheetBenchRunnerReport = { instructionMaxChars: number | null; repairAttempts: number; refreshExcelCaches?: boolean; + allowStablePendingCaches?: boolean; selectedTaskCount: number; }; budget: { @@ -474,6 +525,7 @@ export async function runStagedSpreadsheetBench(options: SpreadsheetBenchRunnerO instructionMaxChars: (options.modelBatchSize ?? 1) > 1 ? BATCH_INSTRUCTION_MAX_CHARS : null, repairAttempts: Math.max(0, Math.min(3, Math.trunc(options.modelRepairAttempts ?? 0))), refreshExcelCaches: options.refreshExcelCaches === true, + allowStablePendingCaches: options.allowStablePendingCaches === true, selectedTaskCount: selectedTasks.length, }, budget: { @@ -544,6 +596,7 @@ async function runTask( modelSnapshotMaxCellChars: options.modelSnapshotMaxCellChars, modelRepairAttempts: options.modelRepairAttempts, refreshExcelCaches: options.refreshExcelCaches, + allowStablePendingCaches: options.allowStablePendingCaches, batchedModelPlan, }); let emitted: string | ModelCandidateEmission; @@ -796,6 +849,7 @@ function emitCandidateWorkbook(args: { modelSnapshotMaxCellChars?: number; modelRepairAttempts?: number; refreshExcelCaches?: boolean; + allowStablePendingCaches?: boolean; batchedModelPlan?: BatchedModelPlan; }): Promise | string { const firstInput = args.agent.inputFiles[0]; @@ -855,6 +909,7 @@ async function emitNodeAgentWorkbookCandidate(args: { modelSnapshotMaxCells?: number; modelRepairAttempts?: number; refreshExcelCaches?: boolean; + allowStablePendingCaches?: boolean; }): Promise { if (!args.model) throw new Error(`nodeagent-workbook requires options.model: ${args.agent.taskId}`); const started = Date.now(); @@ -874,10 +929,11 @@ async function emitNodeAgentWorkbookCandidate(args: { finalizeCandidate: ({ candidateWorkbookPath, beforeSha256 }: { candidateWorkbookPath: string; beforeSha256: string; - }) => refreshCandidateExcelCaches({ + }) => finalizeCandidateExcelCaches({ candidateWorkbookPath, beforeSha256, receiptPath: join(args.taskOutDir, "candidate-finalization.json"), + allowStablePendingCaches: args.allowStablePendingCaches === true, }), } : {}), @@ -934,15 +990,23 @@ async function emitNodeAgentWorkbookCandidate(args: { return { path: args.target, modelPlanningMs, model: modelInfo }; } -function refreshCandidateExcelCaches(args: { +function finalizeCandidateExcelCaches(args: { candidateWorkbookPath: string; beforeSha256: string; receiptPath: string; -}): SpreadsheetBenchCandidateFinalizationReceipt { + allowStablePendingCaches: boolean; +}): SpreadsheetBenchExcelFinalizationReceipt { const script = resolve("scripts", "spreadsheetbench-refresh-excel.py"); const result = spawnSync( process.env.PYTHON?.trim() || "python", - [script, "--file", args.candidateWorkbookPath, "--receipt", args.receiptPath], + [ + script, + "--file", + args.candidateWorkbookPath, + "--receipt", + args.receiptPath, + ...(args.allowStablePendingCaches ? ["--allow-stable-pending"] : []), + ], { cwd: process.cwd(), encoding: "utf8", @@ -952,37 +1016,171 @@ function refreshCandidateExcelCaches(args: { }, ); if (result.error) { - throw new Error(`SpreadsheetBench Excel cache refresh failed to start: ${result.error.message}`); + throw new Error(`SpreadsheetBench Excel finalization failed to start: ${result.error.message}`); } if (result.status !== 0) { const detail = [result.stderr, result.stdout].filter(Boolean).join(" ").replace(/\s+/g, " ").trim().slice(0, 1_000); - throw new Error(`SpreadsheetBench Excel cache refresh failed (${String(result.status)}): ${detail}`); + throw new Error(`SpreadsheetBench Excel finalization failed (${String(result.status)}): ${detail}`); } - const refresh = readJson<{ + return readCandidateExcelFinalizationReceipt(args); +} + +export function readCandidateExcelFinalizationReceipt(args: { + candidateWorkbookPath: string; + beforeSha256: string; + receiptPath: string; +}): SpreadsheetBenchExcelFinalizationReceipt { + const receiptContent = readFileSync(args.receiptPath); + const refresh = JSON.parse(receiptContent.toString("utf8")) as { + schema?: number; engine?: string; + policy?: { fallback?: string }; + workbookCount?: number; records?: Array<{ status?: string; + reason?: string; beforeSha256?: string; afterSha256?: string; + changed?: boolean; formulaCellCount?: number; cacheWriteMode?: string; + calculationStates?: { + beforeCacheRead?: number; + afterCacheRead?: number; + }; + calculationStability?: { + mode?: unknown; + passed?: unknown; + requiredIdenticalReads?: unknown; + observedIdenticalReads?: unknown; + observedReads?: unknown; + sampleIntervalMs?: unknown; + timeoutMs?: unknown; + }; + formulaTopology?: { + safe?: boolean; + supportedTypes?: unknown; + counts?: unknown; + cacheTargetCellCount?: unknown; + unsupported?: unknown; + }; + formulaTopologyPreservation?: { + matched?: unknown; + beforeSha256?: unknown; + afterSha256?: unknown; + formulaElementCount?: unknown; + }; }>; - }>(args.receiptPath); + }; + const engine = refresh.engine?.trim() || ""; + if (refresh.schema !== 1 || refresh.workbookCount !== 1) { + throw new Error("SpreadsheetBench Excel finalization receipt has an invalid single-workbook schema"); + } + if (!engine || /libre\s*office|soffice/i.test(engine) || refresh.policy?.fallback !== "none") { + throw new Error("SpreadsheetBench Excel finalization receipt does not prove the no-fallback Excel policy"); + } + if (refresh.records?.length !== 1) { + throw new Error("SpreadsheetBench Excel finalization receipt must contain exactly one record"); + } const record = refresh.records?.[0]; - if (!record || record.status !== "refreshed") { - throw new Error("SpreadsheetBench Excel cache refresh receipt is missing a refreshed record"); + if (!record || !EXCEL_FINALIZATION_STATUSES.has(record.status as SpreadsheetBenchExcelFinalizationStatus)) { + throw new Error("SpreadsheetBench Excel finalization receipt has a non-canonical terminal status"); } - if (record.beforeSha256 !== args.beforeSha256 || !record.afterSha256) { - throw new Error("SpreadsheetBench Excel cache refresh receipt hash boundary is invalid"); + const status = record.status as SpreadsheetBenchExcelFinalizationStatus; + const reason = record.reason?.trim() || ""; + if (!reason) { + throw new Error("SpreadsheetBench Excel finalization receipt is missing its terminal reason"); + } + if ( + !isSha256(record.beforeSha256) + || record.beforeSha256 !== args.beforeSha256 + || !isSha256(record.afterSha256) + || typeof record.changed !== "boolean" + || record.changed !== (record.beforeSha256 !== record.afterSha256) + ) { + throw new Error("SpreadsheetBench Excel finalization receipt hash boundary is invalid"); + } + const candidateSha256 = createHash("sha256").update(readFileSync(args.candidateWorkbookPath)).digest("hex"); + if (candidateSha256 !== record.afterSha256) { + throw new Error("SpreadsheetBench Excel finalization receipt does not match the candidate workbook"); + } + if ( + (status === "not_required" || status.startsWith("preserved_")) + && (record.changed || record.beforeSha256 !== record.afterSha256) + ) { + throw new Error("SpreadsheetBench non-mutating finalization evidence must report unchanged identical hashes"); + } + + const calculationStates = parseCalculationStates(record.calculationStates); + const formulaTopology = parseFormulaTopology(record.formulaTopology); + const calculationStability = status === "completed_stable_pending" + ? parseCalculationStability(record.calculationStability) + : undefined; + const formulaTopologyPreservation = parseFormulaTopologyPreservation(record.formulaTopologyPreservation); + if (status === "completed") { + if (!Number.isInteger(record.formulaCellCount) || Number(record.formulaCellCount) <= 0) { + throw new Error("SpreadsheetBench completed finalization requires a positive formula cell count"); + } + if (!calculationStates || calculationStates.beforeCacheRead !== 0 || calculationStates.afterCacheRead !== 0) { + throw new Error("SpreadsheetBench completed finalization requires Excel calculation state 0 before and after the cache read"); + } + if (!formulaTopology?.safe) { + throw new Error("SpreadsheetBench completed finalization requires supported formula topology evidence"); + } + if (!formulaTopologyPreservation || formulaTopologyPreservation.formulaElementCount !== record.formulaCellCount) { + throw new Error("SpreadsheetBench completed finalization requires byte-identical formula-topology preservation evidence"); + } + } + if (status === "completed_stable_pending") { + if (!Number.isInteger(record.formulaCellCount) || Number(record.formulaCellCount) <= 0) { + throw new Error("SpreadsheetBench stable-pending finalization requires a positive formula cell count"); + } + if (!calculationStates || (calculationStates.beforeCacheRead === 0 && calculationStates.afterCacheRead === 0)) { + throw new Error("SpreadsheetBench stable-pending finalization requires a nonzero Excel calculation state"); + } + if (!formulaTopology?.safe || !calculationStability) { + throw new Error("SpreadsheetBench stable-pending finalization requires safe topology and stability evidence"); + } + if (!formulaTopologyPreservation || formulaTopologyPreservation.formulaElementCount !== record.formulaCellCount) { + throw new Error("SpreadsheetBench stable-pending finalization requires byte-identical formula-topology preservation evidence"); + } + } + if (status === "not_required" && (record.formulaCellCount !== 0 || formulaTopology?.safe !== true)) { + throw new Error("SpreadsheetBench not_required finalization requires safe zero-formula topology evidence"); + } + if (status === "preserved_pending") { + if (!Number.isInteger(record.formulaCellCount) || Number(record.formulaCellCount) <= 0) { + throw new Error("SpreadsheetBench preserved_pending finalization requires a positive formula cell count"); + } + if (!calculationStates || (calculationStates.beforeCacheRead === 0 && calculationStates.afterCacheRead === 0)) { + throw new Error("SpreadsheetBench preserved_pending finalization requires a nonzero Excel calculation state"); + } + if (formulaTopology?.safe !== true) { + throw new Error("SpreadsheetBench preserved_pending finalization requires supported formula topology evidence"); + } + validateRejectedCalculationStability(record.calculationStability); + } + if (status === "preserved_unsupported") { + if (!Number.isInteger(record.formulaCellCount) || Number(record.formulaCellCount) <= 0) { + throw new Error("SpreadsheetBench preserved_unsupported finalization requires a positive formula cell count"); + } + if (formulaTopology?.safe !== false || !formulaTopology.unsupported?.length) { + throw new Error("SpreadsheetBench preserved_unsupported finalization requires unsafe topology evidence"); + } } - const receiptContent = readFileSync(args.receiptPath); return { - engine: refresh.engine?.trim() || "spreadsheetbench-excel-cache-refresh", + engine, + status, + reason, beforeSha256: record.beforeSha256, afterSha256: record.afterSha256, - changed: record.beforeSha256 !== record.afterSha256, + changed: record.changed, ...(typeof record.formulaCellCount === "number" ? { formulaCellCount: record.formulaCellCount } : {}), ...(record.cacheWriteMode ? { cacheWriteMode: record.cacheWriteMode } : {}), + ...(calculationStates ? { calculationStates } : {}), + ...(formulaTopology ? { formulaTopology } : {}), + ...(calculationStability ? { calculationStability } : {}), + ...(formulaTopologyPreservation ? { formulaTopologyPreservation } : {}), receipt: { path: resolve(args.receiptPath), sha256: createHash("sha256").update(receiptContent).digest("hex"), @@ -991,6 +1189,168 @@ function refreshCandidateExcelCaches(args: { }; } +function parseCalculationStability(value: { + mode?: unknown; + passed?: unknown; + requiredIdenticalReads?: unknown; + observedIdenticalReads?: unknown; + observedReads?: unknown; + sampleIntervalMs?: unknown; + timeoutMs?: unknown; +} | undefined): SpreadsheetBenchExcelFinalizationReceipt["calculationStability"] | undefined { + if (value === undefined) return undefined; + if ( + value.mode !== "stable_pending_opt_in" + || value.passed !== true + || !Number.isInteger(value.requiredIdenticalReads) + || Number(value.requiredIdenticalReads) < 3 + || !Number.isInteger(value.observedIdenticalReads) + || Number(value.observedIdenticalReads) < Number(value.requiredIdenticalReads) + || !Number.isInteger(value.observedReads) + || Number(value.observedReads) < Number(value.observedIdenticalReads) + || !Number.isInteger(value.sampleIntervalMs) + || Number(value.sampleIntervalMs) < 1 + || !Number.isInteger(value.timeoutMs) + || Number(value.timeoutMs) < Number(value.sampleIntervalMs) + ) { + throw new Error("SpreadsheetBench Excel finalization receipt has invalid stable-pending evidence"); + } + return { + mode: "stable_pending_opt_in", + passed: true, + requiredIdenticalReads: Number(value.requiredIdenticalReads), + observedIdenticalReads: Number(value.observedIdenticalReads), + observedReads: Number(value.observedReads), + sampleIntervalMs: Number(value.sampleIntervalMs), + timeoutMs: Number(value.timeoutMs), + }; +} + +function validateRejectedCalculationStability(value: { + mode?: unknown; + passed?: unknown; + requiredIdenticalReads?: unknown; + observedIdenticalReads?: unknown; + observedReads?: unknown; + sampleIntervalMs?: unknown; + timeoutMs?: unknown; +} | undefined): void { + if (value === undefined) return; + if ( + value.mode !== "stable_pending_opt_in" + || value.passed !== false + || !Number.isInteger(value.requiredIdenticalReads) + || Number(value.requiredIdenticalReads) < 3 + || !Number.isInteger(value.observedIdenticalReads) + || Number(value.observedIdenticalReads) < 0 + || Number(value.observedIdenticalReads) >= Number(value.requiredIdenticalReads) + || !Number.isInteger(value.observedReads) + || Number(value.observedReads) < Number(value.observedIdenticalReads) + || !Number.isInteger(value.sampleIntervalMs) + || Number(value.sampleIntervalMs) < 1 + || !Number.isInteger(value.timeoutMs) + || Number(value.timeoutMs) < Number(value.sampleIntervalMs) + ) { + throw new Error("SpreadsheetBench preserved_pending finalization has invalid rejected stability evidence"); + } +} + +function parseFormulaTopologyPreservation(value: { + matched?: unknown; + beforeSha256?: unknown; + afterSha256?: unknown; + formulaElementCount?: unknown; +} | undefined): SpreadsheetBenchExcelFinalizationReceipt["formulaTopologyPreservation"] | undefined { + if (value === undefined) return undefined; + if ( + value.matched !== true + || !isSha256(value.beforeSha256) + || value.beforeSha256 !== value.afterSha256 + || !Number.isInteger(value.formulaElementCount) + || Number(value.formulaElementCount) <= 0 + ) { + throw new Error("SpreadsheetBench Excel finalization receipt has invalid formula-topology preservation evidence"); + } + return { + matched: true, + beforeSha256: value.beforeSha256, + afterSha256: value.afterSha256 as string, + formulaElementCount: Number(value.formulaElementCount), + }; +} + +function parseCalculationStates(value: { + beforeCacheRead?: number; + afterCacheRead?: number; +} | undefined): SpreadsheetBenchExcelFinalizationReceipt["calculationStates"] | undefined { + if (value === undefined) return undefined; + if (!Number.isInteger(value.beforeCacheRead) || !Number.isInteger(value.afterCacheRead)) { + throw new Error("SpreadsheetBench Excel finalization receipt has invalid calculation-state evidence"); + } + return { + beforeCacheRead: value.beforeCacheRead as number, + afterCacheRead: value.afterCacheRead as number, + }; +} + +function parseFormulaTopology(value: { + safe?: boolean; + supportedTypes?: unknown; + counts?: unknown; + cacheTargetCellCount?: unknown; + unsupported?: unknown; +} | undefined): SpreadsheetBenchExcelFormulaTopology | undefined { + if (value === undefined) return undefined; + if (typeof value.safe !== "boolean") { + throw new Error("SpreadsheetBench Excel finalization receipt has invalid formula-topology evidence"); + } + const supportedTypes = Array.isArray(value.supportedTypes) + && value.supportedTypes.every((item) => typeof item === "string") + ? value.supportedTypes as string[] + : undefined; + const counts = isNumericRecord(value.counts) ? value.counts : undefined; + const cacheTargetCellCount = Number.isInteger(value.cacheTargetCellCount) + && Number(value.cacheTargetCellCount) >= 0 + ? Number(value.cacheTargetCellCount) + : undefined; + const unsupported = Array.isArray(value.unsupported) + && value.unsupported.every((item) => isRecord(item)) + ? value.unsupported as Array> + : undefined; + if (value.supportedTypes !== undefined && !supportedTypes) { + throw new Error("SpreadsheetBench Excel finalization receipt has invalid supported topology types"); + } + if (value.counts !== undefined && !counts) { + throw new Error("SpreadsheetBench Excel finalization receipt has invalid topology counts"); + } + if (value.cacheTargetCellCount !== undefined && cacheTargetCellCount === undefined) { + throw new Error("SpreadsheetBench Excel finalization receipt has invalid cache-target count"); + } + if (value.unsupported !== undefined && !unsupported) { + throw new Error("SpreadsheetBench Excel finalization receipt has invalid unsupported topology detail"); + } + return { + safe: value.safe, + ...(supportedTypes ? { supportedTypes } : {}), + ...(counts ? { counts } : {}), + ...(cacheTargetCellCount !== undefined ? { cacheTargetCellCount } : {}), + ...(unsupported ? { unsupported } : {}), + }; +} + +function isSha256(value: unknown): value is string { + return typeof value === "string" && /^[a-f0-9]{64}$/.test(value); +} + +function isNumericRecord(value: unknown): value is Record { + return isRecord(value) + && Object.values(value).every((item) => typeof item === "number" && Number.isInteger(item) && item >= 0); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + type AgentWorkspace = { root: string; agentDir: string; diff --git a/src/eval/spreadsheetBenchStructuralRepair.ts b/src/eval/spreadsheetBenchStructuralRepair.ts new file mode 100644 index 00000000..7ad5dbff --- /dev/null +++ b/src/eval/spreadsheetBenchStructuralRepair.ts @@ -0,0 +1,221 @@ +import { execFileSync } from "node:child_process"; +import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import type { WorkbookObservedCell } from "../nodeagent/skills/spreadsheet/workbookTaskIntelligence"; + +export const SPREADSHEETBENCH_STRUCTURAL_REPAIR_SCHEMA = 1 as const; + +export type SpreadsheetBenchStructuralFormulaRepair = { + sheet: string; + cell: string; + formula: string; +}; + +export type SpreadsheetBenchStructuralRepairPlan = { + schema: typeof SPREADSHEETBENCH_STRUCTURAL_REPAIR_SCHEMA; + status: "complete"; + basis: "visible_workbook_invariants"; + repairId: string; + kind: "insert_missing_selector_row"; + sheet: string; + insertRow: number; + labelCell: string; + label: string; + selectorCell: string; + selectorValue: number; + formulaSearch: string; + formulaReplace: string; + expectedFormulaReplacementCount: number; + formulaRepairs: SpreadsheetBenchStructuralFormulaRepair[]; + operationCount: number; + evidence: string[]; +}; + +export type SpreadsheetBenchStructuralRepairReceipt = { + schema: 1; + backend: "excel_com"; + status: "completed"; + workbookPath: string; + repairIds: string[]; + insertedRowCount: number; + formulaReplacementCount: number; + explicitFormulaRepairCount: number; + calculationPasses: number; +}; + +export function detectSpreadsheetBenchStructuralRepair(args: { + instruction: string; + sheetNames: string[]; + cells: WorkbookObservedCell[]; +}): SpreadsheetBenchStructuralRepairPlan | undefined { + if (!/\b(?:deleted|missing)\s+rows?\b/i.test(args.instruction) + || !/\b(?:audit|fix|repair)\b/i.test(args.instruction)) return undefined; + const selectorMatch = args.instruction.match(/active\s+scenario\s+case\s+selector\s+value\s+is\s+([1-9][0-9]*)/i); + const selectorValue = selectorMatch ? Number(selectorMatch[1]) : NaN; + if (!Number.isInteger(selectorValue) || selectorValue < 1) return undefined; + + const brokenBySheet = new Map(); + for (const cell of args.cells) { + if (!cell.formula || !/\bCHOOSE\s*\(\s*#REF!\s*,/i.test(cell.formula)) continue; + const current = brokenBySheet.get(cell.sheet) ?? []; + current.push(cell); + brokenBySheet.set(cell.sheet, current); + } + const candidate = [...brokenBySheet.entries()] + .sort((left, right) => right[1].length - left[1].length || left[0].localeCompare(right[0]))[0]; + if (!candidate || candidate[1].length < 10) return undefined; + const [sheet, brokenChooseCells] = candidate; + const sheetCells = args.cells.filter((cell) => cell.sheet === sheet); + const title = sheetCells + .filter((cell) => /add[- ]on acquisitions|mergers?\s*&\s*acquisitions|\bm\s*&\s*a\b/i.test(`${cell.value ?? ""} ${cell.formula ?? ""}`)) + .map((cell) => ({ cell, row: addressRow(cell.address) })) + .filter((entry): entry is { cell: WorkbookObservedCell; row: number } => entry.row !== undefined && entry.row >= 3) + .sort((left, right) => left.row - right.row)[0]; + if (!title || title.row > 12) return undefined; + const insertRow = title.row - 1; + const labelCell = `B${insertRow}`; + const selectorCell = `C${insertRow}`; + const occupied = new Set(sheetCells.map((cell) => normalizeCellAddress(cell.address))); + if (occupied.has(labelCell) || occupied.has(selectorCell)) return undefined; + + const formulaRepairs = detectExternalSheetFormulaRepairs(args.cells, args.sheetNames); + const operationCount = 1 + 2 + brokenChooseCells.length + formulaRepairs.length; + return { + schema: SPREADSHEETBENCH_STRUCTURAL_REPAIR_SCHEMA, + status: "complete", + basis: "visible_workbook_invariants", + repairId: `restore-selector-row-${stableIdentifier(sheet)}-${insertRow}`, + kind: "insert_missing_selector_row", + sheet, + insertRow, + labelCell, + label: "Case", + selectorCell, + selectorValue, + formulaSearch: "CHOOSE(#REF!,", + formulaReplace: `CHOOSE($C$${insertRow},`, + expectedFormulaReplacementCount: brokenChooseCells.length, + formulaRepairs, + operationCount, + evidence: [ + `${brokenChooseCells.length} formulas on ${sheet} share a deleted CHOOSE selector reference.`, + `${labelCell}:${selectorCell} is empty immediately before the visible model title row ${title.row}.`, + `The task supplies active scenario selector value ${selectorValue}.`, + `${formulaRepairs.length} external-style sheet reference(s) resolve uniquely to existing workbook sheets.`, + ], + }; +} + +export function applySpreadsheetBenchStructuralRepairs(args: { + workbookPath: string; + repairs: SpreadsheetBenchStructuralRepairPlan[]; + timeoutMs?: number; +}): SpreadsheetBenchStructuralRepairReceipt | undefined { + if (args.repairs.length === 0) return undefined; + if (process.platform !== "win32") { + throw new Error("Spreadsheet structural repair requires the Windows Excel COM backend"); + } + const workbookPath = resolve(args.workbookPath); + if (!existsSync(workbookPath)) throw new Error(`Spreadsheet structural repair workbook does not exist: ${workbookPath}`); + const scriptPath = resolve(process.cwd(), "scripts", "spreadsheetbench-structural-repair.ps1"); + if (!existsSync(scriptPath)) throw new Error(`Spreadsheet structural repair script is missing: ${scriptPath}`); + + const root = mkdtempSync(join(tmpdir(), "noderoom-structural-repair-")); + const planPath = join(root, "plan.json"); + try { + writeFileSync(planPath, `${JSON.stringify({ schema: 1, repairs: args.repairs }, null, 2)}\n`, "utf8"); + const stdout = execFileSync("powershell.exe", [ + "-NoLogo", + "-NoProfile", + "-NonInteractive", + "-ExecutionPolicy", + "Bypass", + "-File", + scriptPath, + "-WorkbookPath", + workbookPath, + "-PlanPath", + planPath, + ], { + encoding: "utf8", + timeout: Math.max(30_000, Math.trunc(args.timeoutMs ?? 180_000)), + windowsHide: true, + maxBuffer: 4 * 1024 * 1024, + }).trim(); + const line = stdout.split(/\r?\n/).map((value) => value.trim()).filter(Boolean).at(-1); + if (!line) throw new Error("Spreadsheet structural repair returned no receipt"); + const receipt = JSON.parse(line) as SpreadsheetBenchStructuralRepairReceipt; + const expectedIds = args.repairs.map((repair) => repair.repairId).sort(); + const actualIds = [...(receipt.repairIds ?? [])].sort(); + if (receipt.schema !== 1 || receipt.status !== "completed" || JSON.stringify(actualIds) !== JSON.stringify(expectedIds)) { + throw new Error(`Spreadsheet structural repair returned an invalid receipt: ${line}`); + } + return receipt; + } finally { + rmSync(root, { recursive: true, force: true }); + } +} + +function detectExternalSheetFormulaRepairs( + cells: WorkbookObservedCell[], + sheetNames: string[], +): SpreadsheetBenchStructuralFormulaRepair[] { + const repairs: SpreadsheetBenchStructuralFormulaRepair[] = []; + for (const cell of cells) { + if (!cell.formula || !/\[[0-9]+\]/.test(cell.formula)) continue; + let changed = false; + const formula = cell.formula.replace(/'\[[0-9]+\]([^']+)'!/g, (token, externalName: string) => { + const match = nearestSheetName(externalName, sheetNames); + if (!match) return token; + changed = true; + return `'${match.replace(/'/g, "''")}'!`; + }); + if (changed && formula !== cell.formula) { + repairs.push({ sheet: cell.sheet, cell: normalizeCellAddress(cell.address), formula: `=${formula.replace(/^=/, "")}` }); + } + } + return repairs.sort((left, right) => left.sheet.localeCompare(right.sheet) || left.cell.localeCompare(right.cell)); +} + +function nearestSheetName(candidate: string, sheetNames: string[]): string | undefined { + const normalized = normalizeSheetName(candidate); + const ranked = sheetNames + .map((sheet) => ({ sheet, distance: editDistance(normalized, normalizeSheetName(sheet)) })) + .sort((left, right) => left.distance - right.distance || left.sheet.localeCompare(right.sheet)); + if (!ranked[0] || ranked[0].distance > 1 || ranked[1]?.distance === ranked[0].distance) return undefined; + return ranked[0].sheet; +} + +function editDistance(left: string, right: string): number { + const previous = Array.from({ length: right.length + 1 }, (_value, index) => index); + for (let leftIndex = 1; leftIndex <= left.length; leftIndex += 1) { + const current = [leftIndex]; + for (let rightIndex = 1; rightIndex <= right.length; rightIndex += 1) { + current[rightIndex] = Math.min( + current[rightIndex - 1] + 1, + previous[rightIndex] + 1, + previous[rightIndex - 1] + (left[leftIndex - 1] === right[rightIndex - 1] ? 0 : 1), + ); + } + previous.splice(0, previous.length, ...current); + } + return previous[right.length]; +} + +function normalizeSheetName(value: string): string { + return value.trim().replace(/\s+/g, " ").toLowerCase(); +} + +function addressRow(address: string): number | undefined { + const match = normalizeCellAddress(address).match(/^[A-Z]{1,3}([1-9][0-9]*)$/); + return match ? Number(match[1]) : undefined; +} + +function normalizeCellAddress(value: string): string { + return value.trim().replace(/\$/g, "").toUpperCase(); +} + +function stableIdentifier(value: string): string { + return value.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "").slice(0, 48) || "sheet"; +} diff --git a/src/eval/spreadsheetBenchWorkbookEmitter.ts b/src/eval/spreadsheetBenchWorkbookEmitter.ts index ca726633..cc29f912 100644 --- a/src/eval/spreadsheetBenchWorkbookEmitter.ts +++ b/src/eval/spreadsheetBenchWorkbookEmitter.ts @@ -3,16 +3,20 @@ import { copyFileSync, readFileSync, renameSync, unlinkSync, writeFileSync } fro import { basename, dirname, join, posix, resolve } from "node:path"; import JSZip from "jszip"; import { SaxesParser, type SaxesTagNS } from "saxes"; +import { normalizeSpreadsheetFontColor } from "../shared/spreadsheetFontColor"; export type SpreadsheetBenchWorkbookCellPatch = { sheet: string; address: string; - kind: "clear" | "formula" | "value"; + kind: "clear" | "formula" | "style" | "value"; formula?: string; hasCachedResult?: boolean; cachedResult?: unknown; value?: unknown; numFmt?: string; + fontColor?: string; + fontColorTheme?: number; + fontColorTint?: number; }; type WorkbookSheetPart = { @@ -141,11 +145,14 @@ export async function emitSpreadsheetBenchWorkbookCandidate(args: { class WorkbookStylesEditor { private readonly sourceXml: string; private readonly sourceXfs: string[]; + private readonly sourceFonts: string[]; private readonly formatCodeById = new Map(BUILTIN_NUMBER_FORMATS); private readonly formatIdByCode = new Map(); private readonly pendingNumberFormats: Array<{ id: number; code: string }> = []; + private readonly pendingFonts: string[] = []; private readonly pendingXfs: string[] = []; private readonly styleCache = new Map(); + private readonly fontCache = new Map(); private nextNumberFormatId: number; constructor(xml: string) { @@ -153,6 +160,12 @@ class WorkbookStylesEditor { const nodes = indexXmlSpans(xml); const cellXfs = nodes.find((node) => node.local === "cellXfs"); if (!cellXfs) throw new Error("Workbook styles XML is missing cellXfs"); + const fonts = nodes.find((node) => node.local === "fonts"); + if (!fonts) throw new Error("Workbook styles XML is missing fonts"); + this.sourceFonts = nodes + .filter((node) => node.local === "font" && node.parent === fonts) + .map((node) => xml.slice(node.start, node.end)); + if (this.sourceFonts.length === 0) throw new Error("Workbook styles XML has no font entries"); this.sourceXfs = nodes .filter((node) => node.local === "xf" && node.parent === cellXfs) .map((node) => xml.slice(node.start, node.end)); @@ -169,20 +182,37 @@ class WorkbookStylesEditor { this.nextNumberFormatId = Math.max(164, ...this.formatCodeById.keys()) + 1; } - ensureNumberFormatStyle(sourceStyleIndex: number, formatCode: string): number { + ensureCellStyle(sourceStyleIndex: number, changes: { + numFmt?: string; + fontColor?: string; + fontColorTheme?: number; + fontColorTint?: number; + }): number { const sourceXf = this.sourceXfs[sourceStyleIndex]; if (!sourceXf) throw new Error(`Workbook cell references missing style index ${sourceStyleIndex}`); const sourceXfNode = indexXmlSpans(sourceXf).find((node) => node.local === "xf"); if (!sourceXfNode) throw new Error(`Workbook style ${sourceStyleIndex} is malformed`); const sourceFormatId = Number(sourceXfNode.attrs.numFmtId ?? 0); - if (this.formatCodeById.get(sourceFormatId) === formatCode) return sourceStyleIndex; + const sourceFontId = Number(sourceXfNode.attrs.fontId ?? 0); + const normalizedFontColor = changes.fontColor === undefined ? undefined : normalizeSpreadsheetFontColor(changes.fontColor); + if (changes.fontColor !== undefined && !normalizedFontColor) throw new Error(`Invalid workbook font color ${JSON.stringify(changes.fontColor)}`); + const formatId = changes.numFmt === undefined ? sourceFormatId : this.ensureNumberFormat(changes.numFmt); + const fontId = normalizedFontColor === undefined && changes.fontColorTheme === undefined + ? sourceFontId + : this.ensureFontColor(sourceFontId, { + rgb: normalizedFontColor, + theme: changes.fontColorTheme, + tint: changes.fontColorTint, + }); + if (formatId === sourceFormatId && fontId === sourceFontId) return sourceStyleIndex; - const cacheKey = `${sourceStyleIndex}\u0000${formatCode}`; + const cacheKey = `${sourceStyleIndex}\u0000${formatId}\u0000${fontId}`; const cached = this.styleCache.get(cacheKey); if (cached !== undefined) return cached; - const formatId = this.ensureNumberFormat(formatCode); let nextXf = setElementAttribute(sourceXf, "numFmtId", String(formatId)); nextXf = setElementAttribute(nextXf, "applyNumberFormat", formatId === 0 ? "0" : "1"); + nextXf = setElementAttribute(nextXf, "fontId", String(fontId)); + if (fontId !== sourceFontId) nextXf = setElementAttribute(nextXf, "applyFont", "1"); const styleIndex = this.sourceXfs.length + this.pendingXfs.length; this.pendingXfs.push(nextXf); this.styleCache.set(cacheKey, styleIndex); @@ -222,6 +252,18 @@ class WorkbookStylesEditor { } } + if (this.pendingFonts.length > 0) { + const nodes = indexXmlSpans(xml); + const fonts = nodes.find((node) => node.local === "fonts"); + if (!fonts || fonts.selfClosing) throw new Error("Workbook styles XML has no writable fonts collection"); + const count = nodes.filter((node) => node.local === "font" && node.parent === fonts).length; + const opening = setXmlAttribute(xml.slice(fonts.start, fonts.openEnd), "count", String(count + this.pendingFonts.length)); + xml = applyXmlSplices(xml, [ + { start: fonts.start, end: fonts.openEnd, text: opening }, + { start: fonts.closeStart, end: fonts.closeStart, text: this.pendingFonts.join("") }, + ]); + } + if (this.pendingXfs.length > 0) { const nodes = indexXmlSpans(xml); const cellXfs = nodes.find((node) => node.local === "cellXfs"); @@ -245,6 +287,28 @@ class WorkbookStylesEditor { this.formatCodeById.set(id, formatCode); return id; } + + private ensureFontColor( + sourceFontId: number, + color: { rgb?: string; theme?: number; tint?: number }, + ): number { + const sourceFont = this.sourceFonts[sourceFontId]; + if (!sourceFont) throw new Error(`Workbook style references missing font index ${sourceFontId}`); + if (color.theme !== undefined && (!Number.isInteger(color.theme) || color.theme < 0)) { + throw new Error(`Invalid workbook font theme ${JSON.stringify(color.theme)}`); + } + const token = color.theme === undefined + ? { rgb: color.rgb } + : { theme: color.theme, ...(color.tint === undefined ? {} : { tint: color.tint }) }; + if (fontColorTokenMatches(sourceFont, token)) return sourceFontId; + const cacheKey = `${sourceFontId}\u0000${JSON.stringify(token)}`; + const cached = this.fontCache.get(cacheKey); + if (cached !== undefined) return cached; + const fontId = this.sourceFonts.length + this.pendingFonts.length; + this.pendingFonts.push(setFontColorToken(sourceFont, token)); + this.fontCache.set(cacheKey, fontId); + return fontId; + } } function patchWorksheetCells( @@ -383,7 +447,7 @@ function prepareFormulaGroupSplices( nodes: XmlNodeSpan[], patches: SpreadsheetBenchWorkbookCellPatch[], ): XmlSplice[] { - const patchAddresses = new Set(patches.map((patch) => patch.address)); + const patchAddresses = new Set(patches.filter((patch) => patch.kind !== "style").map((patch) => patch.address)); const formulaNodes = nodes.filter((node) => node.local === "f" && node.parent?.local === "c"); for (const formula of formulaNodes.filter((node) => node.attrs.t === "array" || node.attrs.t === "dataTable")) { @@ -511,19 +575,30 @@ function serializeCellPatch( sourceStyleIndex: number, styles: WorkbookStylesEditor, ): string { + const styleIndex = patch.numFmt === undefined && patch.fontColor === undefined && patch.fontColorTheme === undefined + ? sourceStyleIndex + : styles.ensureCellStyle(sourceStyleIndex, { + numFmt: patch.numFmt, + fontColor: patch.fontColor, + fontColorTheme: patch.fontColorTheme, + fontColorTint: patch.fontColorTint, + }); + if (patch.kind === "style") { + if (!sourceCell) return ``; + let opening = xml.slice(sourceCell.start, sourceCell.openEnd); + opening = setXmlAttribute(opening, "s", String(styleIndex)); + return `${opening}${xml.slice(sourceCell.openEnd, sourceCell.end)}`; + } const formulaElement = patch.kind === "formula" ? sourceFormulaElement(xml, sourceCell, patch) : { opening: "", closing: "" }; - const styleIndex = patch.numFmt === undefined - ? sourceStyleIndex - : styles.ensureNumberFormatStyle(sourceStyleIndex, patch.numFmt); const tagName = sourceCell?.name ?? "c"; let opening = sourceCell ? xml.slice(sourceCell.start, sourceCell.openEnd) : `<${tagName} r="${patch.address}">`; opening = opening.replace(/\s*\/>$/, ">"); opening = setXmlAttribute(opening, "r", patch.address); opening = removeXmlAttribute(opening, "t"); - if (patch.numFmt !== undefined) { + if (patch.numFmt !== undefined || patch.fontColor !== undefined || patch.fontColorTheme !== undefined) { opening = setXmlAttribute(opening, "s", String(styleIndex)); } @@ -601,7 +676,7 @@ function serializedScalar(value: unknown, formulaResult: boolean): { ? value : typeof record?.text === "string" ? record.text - : richText; + : richText ?? structuredCellText(value); if (text === undefined) { throw new Error(`Workbook patch cannot serialize ${Array.isArray(value) ? "an array" : typeof value} as a cell scalar`); } @@ -610,6 +685,29 @@ function serializedScalar(value: unknown, formulaResult: boolean): { : { type: "inlineStr", valueXml: escapeXmlText(text), inlineString: true, textValue: text }; } +function structuredCellText(value: unknown): string | undefined { + if ((!value || typeof value !== "object") && !Array.isArray(value)) return undefined; + const canonical = canonicalJsonValue(value, new Set()); + const text = JSON.stringify(canonical); + if (text.length > 32_767) throw new Error(`Workbook patch structured cell text exceeds Excel's 32767 character limit`); + return text; +} + +function canonicalJsonValue(value: unknown, ancestors: Set): unknown { + if (value === null || typeof value !== "object") return value; + if (value instanceof Date) return value.toISOString(); + if (ancestors.has(value)) throw new Error("Workbook patch structured cell value contains a cycle"); + ancestors.add(value); + try { + if (Array.isArray(value)) return value.map((item) => canonicalJsonValue(item, ancestors)); + return Object.fromEntries(Object.entries(value as Record) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, nested]) => [key, canonicalJsonValue(nested, ancestors)])); + } finally { + ancestors.delete(value); + } +} + function updateWorksheetDimension(xml: string): string { const nodes = indexXmlSpans(xml); const cells = nodes.filter((node) => node.local === "c" && parseAddress(node.attrs.r ?? "")); @@ -720,6 +818,41 @@ function assertWellFormedXml(xml: string, label: string): void { } } +function fontColorTokenMatches( + fontXml: string, + token: { rgb?: string; theme?: number; tint?: number }, +): boolean { + const nodes = indexXmlSpans(fontXml); + const font = nodes.find((node) => node.local === "font"); + const color = nodes.find((node) => node.local === "color" && node.parent === font); + if (!color) return false; + if (token.theme !== undefined) { + return Number(color.attrs.theme) === token.theme + && (token.tint === undefined ? color.attrs.tint === undefined : Number(color.attrs.tint) === token.tint); + } + return normalizeSpreadsheetFontColor(color.attrs.rgb) === token.rgb; +} + +function setFontColorToken( + fontXml: string, + token: { rgb?: string; theme?: number; tint?: number }, +): string { + const nodes = indexXmlSpans(fontXml); + const font = nodes.find((node) => node.local === "font"); + if (!font) throw new Error("Workbook font XML is malformed"); + const color = nodes.find((node) => node.local === "color" && node.parent === font); + if (token.theme === undefined && !token.rgb) throw new Error("Workbook font color token is empty"); + const fragment = token.theme === undefined + ? `` + : ``; + if (color) return applyXmlSplices(fontXml, [{ start: color.start, end: color.end, text: fragment }]); + if (font.selfClosing) { + const opening = fontXml.slice(font.start, font.openEnd).replace(/\s*\/>$/, ">"); + return `${opening}${fragment}`; + } + return applyXmlSplices(fontXml, [{ start: font.closeStart, end: font.closeStart, text: fragment }]); +} + function setElementAttribute(elementXml: string, name: string, value: string): string { const end = elementXml.indexOf(">"); if (end < 0) throw new Error("Malformed XML element"); diff --git a/src/nodeagent/core/runtime.ts b/src/nodeagent/core/runtime.ts index 338d2f1b..5a0ec5bc 100644 --- a/src/nodeagent/core/runtime.ts +++ b/src/nodeagent/core/runtime.ts @@ -150,6 +150,51 @@ function deterministicToolFailureKey(toolName: string, result: unknown): string return JSON.stringify([toolName, error, failureKind, reasonKey]); } +const SCHEMA_PLACEHOLDER_STRINGS = new Set(["string", ""]); + +function schemaPlaceholderArgumentPaths(value: unknown): string[] | null { + const stringLeaves: Array<{ path: string; value: string }> = []; + + const visit = (current: unknown, path: string): void => { + if (typeof current === "string") { + if (current.trim().length > 0) stringLeaves.push({ path: path || "$", value: current }); + return; + } + if (Array.isArray(current)) { + current.forEach((item, index) => visit(item, `${path}[${index}]`)); + return; + } + if (!current || typeof current !== "object") return; + for (const [key, item] of Object.entries(current)) { + visit(item, path ? `${path}.${key}` : key); + } + }; + + visit(value, ""); + if (stringLeaves.length === 0 || stringLeaves.some((leaf) => !SCHEMA_PLACEHOLDER_STRINGS.has(leaf.value))) { + return null; + } + return stringLeaves.map((leaf) => leaf.path); +} + +function schemaPlaceholderArgumentResult(toolName: string, paths: string[]) { + return { + ok: false, + error: "tool_argument_error", + failureKind: "schema_placeholder_arguments", + missingRequiredArgs: [], + issues: paths.map((path) => ({ + path, + code: "schema_placeholder_arguments", + message: "Replace the schema placeholder with a real task-specific value.", + })), + recovery: { + action: "retry_tool_call", + instruction: `Retry ${toolName} with real task-specific argument values instead of schema placeholders such as \"string\" or \"\".`, + }, + }; +} + function toolArgumentErrorResult(toolName: string, issues: Array<{ path: PropertyKey[]; code: string; message: string }>): ToolArgumentErrorResult { const normalized = issues.map((issue) => ({ path: issue.path.map(String).join("."), @@ -680,18 +725,23 @@ export async function runAgent(opts: { const parsed = tool.schema.safeParse(activeCall.args); try { if (parsed.success) { - const coverageFailure = btbPackageCoverageFailure(goal, activeCall.tool, parsed.data); - if (coverageFailure) { - result = coverageFailure; + const placeholderPaths = schemaPlaceholderArgumentPaths(parsed.data); + if (placeholderPaths) { + result = schemaPlaceholderArgumentResult(activeCall.tool, placeholderPaths); } else { - const signal = modelSignal(); - try { - result = await settleWithAbort(tool.execute(parsed.data, rt, { - signal: signal.signal, - deadlineAt: deadlineAt === undefined ? undefined : deadlineAt - reserveMs, - }), signal.signal); - } finally { - signal.cancel(); + const coverageFailure = btbPackageCoverageFailure(goal, activeCall.tool, parsed.data); + if (coverageFailure) { + result = coverageFailure; + } else { + const signal = modelSignal(); + try { + result = await settleWithAbort(tool.execute(parsed.data, rt, { + signal: signal.signal, + deadlineAt: deadlineAt === undefined ? undefined : deadlineAt - reserveMs, + }), signal.signal); + } finally { + signal.cancel(); + } } } } else { diff --git a/src/nodeagent/guardrails/workbookWorkflow.ts b/src/nodeagent/guardrails/workbookWorkflow.ts index 7bbb6109..5f5f60eb 100644 --- a/src/nodeagent/guardrails/workbookWorkflow.ts +++ b/src/nodeagent/guardrails/workbookWorkflow.ts @@ -1,5 +1,6 @@ import type { AgentMessage, ToolCall } from "../core/types"; import type { BlockedTool, HookCtx, NodeAgentHook, StopDecision } from "../core/hooks"; +import { normalizeSpreadsheetFontColor } from "../../shared/spreadsheetFontColor"; const INSPECT_TOOL = "inspect_workbook"; const VERIFY_TOOL = "verify_workbook"; @@ -21,6 +22,7 @@ type WorkbookOperation = { value?: unknown; result?: unknown; numFmt?: string; + fontColor?: string; }; type WorkbookPlan = { @@ -248,7 +250,7 @@ class WorkbookWorkflowState { const requestedTargets = targetSummary(requestedPlan.operations); return blocked( "write_plan_mismatch", - "The managed write must match one passing preflight plan exactly, including artifact, targets, base versions, formulas or values, cached results, and number formats.", + "The managed write must match one passing preflight plan exactly, including artifact, targets, base versions, formulas or values, cached results, number formats, and font colors.", approved ? `Retry write_locked_cells with the approved artifact and at least one unchanged operation from its ${approved.operations.length}-operation plan, or submit a corrected replacement preflight first.` : "Retry with the exact artifact and operation set from one passing preflight, or submit a corrected replacement preflight first.", @@ -392,6 +394,9 @@ function normalizeOperation(value: unknown, artifactId: string): WorkbookOperati : nested && formula ? undefined : ownValue(operation, ["value", "newValue", "new_value", "text", "content"]); const numFmt = stringField(operation, ["numFmt", "num_fmt", "numberFormat", "number_format"]) ?? (nested ? stringField(nested, ["numFmt", "num_fmt", "numberFormat", "number_format"]) : undefined); + const rawFontColor = ownValue(operation, ["fontColor", "font_color"]) + ?? (nested ? ownValue(nested, ["fontColor", "font_color"]) : undefined); + const fontColor = normalizeSpreadsheetFontColor(rawFontColor); const baseVersion = integerValue(ownValue(operation, ["baseVersion", "base_version", "currentVersion", "current_version", "version"])); const normalizedElementId = normalizeElementId(elementId); return [{ @@ -402,6 +407,7 @@ function normalizeOperation(value: unknown, artifactId: string): WorkbookOperati ...(formula && result !== undefined ? { result: normalizeJson(result) } : {}), ...(!formula && scalarValue !== undefined ? { value: normalizeJson(scalarValue) } : {}), ...(numFmt?.trim() ? { numFmt: numFmt.trim() } : {}), + ...(fontColor ? { fontColor } : {}), }]; } @@ -425,7 +431,16 @@ function displayArtifact(value: string): string { } function normalizeElementId(value: string): string { - return value.trim().replace(/\$/g, "").replace(/^'([^']+)'!/, "$1!").replace(/\s*!\s*/g, "!").toUpperCase(); + const trimmed = value.trim(); + const reference = trimmed.match( + /^(?:(?:'((?:[^']|'')+)'|([^!]+?))\s*!\s*)?(\$?[A-Z]{1,3}\$?[1-9][0-9]*)(?:\s*:\s*(\$?[A-Z]{1,3}\$?[1-9][0-9]*))?$/i, + ); + if (!reference) return trimmed; + + const sheet = reference[1]?.replace(/''/g, "'") ?? reference[2]?.trim(); + const start = reference[3].replace(/\$/g, "").toUpperCase(); + const end = reference[4]?.replace(/\$/g, "").toUpperCase(); + return `${sheet ? `${sheet}!` : ""}${start}${end ? `:${end}` : ""}`; } function targetFor(elementId: string, artifactId: string): string { @@ -527,6 +542,7 @@ function operationArgs(operation: WorkbookOperation): Record { ...(Object.prototype.hasOwnProperty.call(operation, "value") ? { value: operation.value } : {}), ...(Object.prototype.hasOwnProperty.call(operation, "result") ? { result: operation.result } : {}), ...(operation.numFmt ? { numFmt: operation.numFmt } : {}), + ...(operation.fontColor ? { fontColor: operation.fontColor } : {}), }; } diff --git a/src/nodeagent/models/adapter.ts b/src/nodeagent/models/adapter.ts index 13c9ce46..4985fd50 100644 --- a/src/nodeagent/models/adapter.ts +++ b/src/nodeagent/models/adapter.ts @@ -149,7 +149,7 @@ type SdkToolSet = Record; type GenerateTextResultAny = any; const NODEAGENT_PROVIDER_NEUTRAL_POLICY = "nodeagent_provider_neutral_free_first_v1" as const; -const DEFAULT_NODEAGENT_GOOGLE_FALLBACK_MODEL = "gemini-3-flash-preview"; +const DEFAULT_NODEAGENT_GOOGLE_FALLBACK_MODEL = "gemini-3.5-flash"; type ProviderNeutralBilling = { /** This describes route pricing, not the account's eventual invoice or free-tier entitlement. */ diff --git a/src/nodeagent/models/convexModel.ts b/src/nodeagent/models/convexModel.ts index d49fa961..da0293d3 100644 --- a/src/nodeagent/models/convexModel.ts +++ b/src/nodeagent/models/convexModel.ts @@ -1248,6 +1248,7 @@ export function toolParameters(toolName: string): JsonObject { }; const scalarWriteKind = { type: "string", enum: ["set", "create", "delete"] }; const resultWriteKind = { type: "string", enum: ["set", "create"] }; + const fontColor = { type: "string", pattern: "^#?(?:[0-9A-Fa-f]{6}|[0-9A-Fa-f]{8})$" }; const managedScalarWriteProperties = { elementId: string, cellId: string, @@ -1268,6 +1269,8 @@ export function toolParameters(toolName: string): JsonObject { num_fmt: string, numberFormat: string, number_format: string, + fontColor, + font_color: fontColor, text: any, content: any, expectedValue: any, @@ -1323,6 +1326,8 @@ export function toolParameters(toolName: string): JsonObject { numFmt: any, numberFormats: any, numberFormat: any, + fontColors: any, + fontColor: any, text: any, content: any, expectedValue: any, @@ -1389,7 +1394,7 @@ export function toolParameters(toolName: string): JsonObject { const stringOrStringArray = { anyOf: [stringArray, string] }; const workbookOperation = { type: "object", - properties: { elementId: string, baseVersion: integer, value: any, formula: string, result: any, numFmt: string }, + properties: { elementId: string, baseVersion: integer, value: any, formula: string, result: any, numFmt: string, fontColor }, required: ["elementId"], }; const schemas: Record = { @@ -1398,6 +1403,11 @@ export function toolParameters(toolName: string): JsonObject { properties: { instruction: string, artifactId: string, query: string, maxCells: integer }, required: ["instruction"], }, + execute_workbook_structure_repair: { + type: "object", + properties: { instruction: string, artifactId: string, repairId: string }, + required: ["instruction", "repairId"], + }, execute_verified_workbook_plan: { type: "object", properties: { diff --git a/src/nodeagent/skills/spreadsheet/cellMutator.ts b/src/nodeagent/skills/spreadsheet/cellMutator.ts index 1d57d535..1916720c 100644 --- a/src/nodeagent/skills/spreadsheet/cellMutator.ts +++ b/src/nodeagent/skills/spreadsheet/cellMutator.ts @@ -13,6 +13,7 @@ import { z } from "zod"; import type { AgentTool, EditOutcome, RoomTools } from "../../core/types"; import type { CellEvidence, CellPayload, CellStatus } from "../../../engine/types"; +import { normalizeSpreadsheetFontColor } from "../../../shared/spreadsheetFontColor"; import { runAlgorithmArtifactFromRoomTools, type AlgorithmArtifact } from "./algorithmArtifacts"; import { BANKER_COACH_TOOLS } from "../bankerCoach/tools"; import { OKF_RETRIEVAL_TOOLS } from "../../retrieval/tools"; @@ -87,7 +88,16 @@ type ParallelField = { type ScalarWriteKind = "set" | "create" | "delete"; type ResultWriteKind = "set" | "create"; type RawManagedOp = Record; -type ScalarManagedOp = { elementId: string; value: unknown; baseVersion?: number; kind?: ScalarWriteKind }; +type ManagedStylePatch = { numFmt?: string; fontColor?: string }; +type ScalarManagedOp = { + elementId: string; + value: unknown; + baseVersion?: number; + kind?: ScalarWriteKind; + fontColor?: string; + stylePatch?: ManagedStylePatch; + preserveFontColor?: boolean; +}; type ScalarManagedOpWithVersion = { elementId: string; value: unknown; baseVersion: number; kind?: ScalarWriteKind }; type ResultManagedOp = ScalarManagedOp & { status: CellStatus; @@ -108,15 +118,16 @@ function normalizeParallelBatchCall(value: unknown, fields: ParallelField[] = [] const elementIds = arrayish(record.elementIds ?? record.cellIds ?? record.ids ?? record.targets ?? record.targetCells ?? record.elementId ?? record.cellId ?? record.id ?? record.cell ?? record.targetCell ?? record.target); const values = arrayish(record.values ?? record.newValues ?? record.results ?? record.value ?? record.newValue ?? record.new_value ?? record.result ?? record.text ?? record.content ?? record.expectedValue); const baseVersions = arrayish(record.baseVersions ?? record.base_versions ?? record.baseVersion ?? record.base_version ?? record.currentVersions ?? record.currentVersion ?? record.versions ?? record.version); - if (!elementIds.length || values.length !== elementIds.length) return record; + const hasStyleOnlyField = fields.some((field) => + (field.opKey === "fontColor" || field.opKey === "numFmt") + && field.inputKeys.some((key) => record[key] !== undefined)); + if (!elementIds.length || (values.length === 0 ? !hasStyleOnlyField : values.length !== elementIds.length)) return record; if (baseVersions.length && baseVersions.length !== 1 && baseVersions.length !== elementIds.length) return record; const kinds = arrayish(record.kinds ?? record.kind); record.ops = elementIds.map((elementId, idx) => { - const op: Record = { - elementId, - value: values[idx], - }; + const op: Record = { elementId }; + if (values.length) op.value = values[idx]; if (baseVersions.length) op.baseVersion = baseVersions.length === 1 ? baseVersions[0] : baseVersions[idx]; if (kinds.length === 1 || kinds.length === elementIds.length) op.kind = kinds.length === 1 ? kinds[0] : kinds[idx]; for (const field of fields) { @@ -149,11 +160,20 @@ function normalizeScalarOp(value: unknown, wrapWorkbookPayload = true): ScalarMa const nextValue = firstDefined(op, ["value", "newValue", "new_value", "result", "text", "content", "expectedValue", "expected_value"]); const formula = firstString(op, ["formula"]); const numFmt = firstString(op, ["numFmt", "num_fmt", "numberFormat", "number_format"]); + const nested = cellPayloadRecord(nextValue); + const fontColorField = normalizedFontColorField(op, nested); + const fontColor = fontColorField.value; if (typeof elementId !== "string" || !elementId.trim()) throw new Error("managed_write_missing_elementId"); const kind = op.kind === "create" || op.kind === "delete" || op.kind === "set" ? op.kind : undefined; - if (nextValue === undefined && formula === undefined && numFmt === undefined && kind !== "delete") throw new Error("managed_write_missing_value"); - const workbookValue = wrapWorkbookPayload && (formula !== undefined || numFmt !== undefined) - ? workbookCellPayload(op, nextValue, formula, numFmt) + if (nextValue === undefined && formula === undefined && numFmt === undefined && !fontColorField.specified && kind !== "delete") throw new Error("managed_write_missing_value"); + const stylePatch = wrapWorkbookPayload && nextValue === undefined && formula === undefined && (numFmt !== undefined || fontColorField.specified) + ? { + ...(numFmt === undefined ? {} : { numFmt }), + ...(fontColor === undefined ? {} : { fontColor }), + } + : undefined; + const workbookValue = wrapWorkbookPayload && !stylePatch && (formula !== undefined || numFmt !== undefined || fontColorField.specified) + ? workbookCellPayload(op, nextValue, formula, numFmt, fontColor) : nextValue; return { ...op, @@ -161,20 +181,47 @@ function normalizeScalarOp(value: unknown, wrapWorkbookPayload = true): ScalarMa value: workbookValue, baseVersion: numericVersion(firstDefined(op, ["baseVersion", "base_version", "currentVersion", "current_version", "version"])), kind, + ...(fontColor === undefined ? {} : { fontColor }), + ...(stylePatch ? { stylePatch } : {}), + ...(!fontColorField.specified && kind !== "create" && kind !== "delete" ? { preserveFontColor: true } : {}), } as ScalarManagedOp; } +function cellPayloadRecord(value: unknown): Record | undefined { + return value && typeof value === "object" && !Array.isArray(value) && Object.prototype.hasOwnProperty.call(value, "value") + ? value as Record + : undefined; +} + +function normalizedFontColorField( + operation: RawManagedOp, + nested?: Record, +): { specified: boolean; value?: string } { + for (const record of [operation, nested]) { + if (!record) continue; + for (const key of ["fontColor", "font_color"]) { + if (!Object.prototype.hasOwnProperty.call(record, key) || record[key] === undefined) continue; + const normalized = normalizeSpreadsheetFontColor(record[key]); + if (!normalized) throw new Error("managed_write_invalid_fontColor"); + return { specified: true, value: normalized }; + } + } + return { specified: false }; +} + function workbookCellPayload( op: RawManagedOp, nextValue: unknown, formula: string | undefined, numFmt: string | undefined, + fontColor: string | undefined, ): CellPayload { const nested = nextValue && typeof nextValue === "object" && !Array.isArray(nextValue) && "value" in nextValue ? nextValue as Record : undefined; const nestedPayload = nested ? { ...nested } : {}; delete nestedPayload.result; + delete nestedPayload.font_color; const rawCachedValue = op.result !== undefined ? op.result : nested?.result ?? nested?.value ?? nextValue ?? ""; @@ -186,6 +233,7 @@ function workbookCellPayload( value: cachedValue, ...(formula === undefined ? {} : { formula }), ...(numFmt === undefined ? {} : { numFmt }), + ...(fontColor === undefined ? {} : { fontColor }), }; } @@ -212,7 +260,7 @@ function addScalarOpIssues(value: unknown, ctx: z.RefinementCtx) { const message = error instanceof Error ? error.message : String(error); ctx.addIssue({ code: z.ZodIssueCode.custom, - path: message.includes("value") ? ["value"] : ["elementId"], + path: message.includes("fontColor") ? ["fontColor"] : message.includes("value") ? ["value"] : ["elementId"], message, }); } @@ -241,6 +289,10 @@ function hasNormalizableBatchOps(value: unknown, fields: ParallelField[] = []) { const opSchema = z.object({ elementId: z.string(), value: z.any(), baseVersion: z.coerce.number().int() }); const cellStatusSchema = z.enum(["empty", "running", "complete", "needs_review", "failed", "gap"]); +const fontColorSchema = z.string().refine( + (value) => normalizeSpreadsheetFontColor(value) !== undefined, + "fontColor must be RRGGBB, #RRGGBB, AARRGGBB, or #AARRGGBB", +); const evidenceSchema = z.object({ id: z.string().optional(), kind: z.enum(["upload", "source", "computed", "manual"]), @@ -274,6 +326,7 @@ function cellPayload(args: { error?: string; normalizedValue?: unknown; formula?: string; + fontColor?: string; review?: CellPayload["review"]; }): CellPayload { return { @@ -283,6 +336,7 @@ function cellPayload(args: { error: args.error, normalizedValue: args.normalizedValue, formula: args.formula, + fontColor: args.fontColor, review: args.review, evidence: args.evidence.map((e, idx) => ({ ...e, @@ -316,6 +370,7 @@ async function reviewedCellPayload(args: { error?: string; normalizedValue?: unknown; formula?: string; + fontColor?: string; }, rt: RoomTools): Promise { if (!rt.okf || !shouldCheckOkfEvidence(args)) return cellPayload(args); const claim = `${args.elementId}: ${compactClaimValue(args.value)}`; @@ -406,6 +461,46 @@ async function withBaseVersions(ops: ScalarManagedOp[], rt: RoomTools, artifactI })); } +async function prepareManagedOps( + ops: ScalarManagedOp[], + rt: RoomTools, + artifactId?: string, +): Promise { + const needCurrent = ops.filter((op) => + (op.kind ?? "set") !== "create" + && (op.baseVersion === undefined || op.stylePatch !== undefined || op.preserveFontColor === true)); + const current = needCurrent.length && typeof rt.readRange === "function" + ? await rt.readRange([...new Set(needCurrent.map((op) => op.elementId))], artifactId) + : []; + const byId = new Map(current.map((cell) => [cell.id, cell])); + return ops.map((op) => { + const cell = byId.get(op.elementId); + const { stylePatch, preserveFontColor, ...prepared } = op; + let value = op.value; + if (stylePatch) { + const existing = cellPayloadRecord(cell?.value); + value = { + ...(existing ?? {}), + value: existing ? existing.value : cell?.value ?? null, + ...stylePatch, + } satisfies CellPayload; + } else if (preserveFontColor) { + const currentFontColor = normalizeSpreadsheetFontColor(cellPayloadRecord(cell?.value)?.fontColor); + if (currentFontColor) { + const proposed = cellPayloadRecord(value); + value = proposed + ? { ...proposed, fontColor: currentFontColor } + : { value, fontColor: currentFontColor } satisfies CellPayload; + } + } + return { + ...prepared, + value, + ...(prepared.baseVersion === undefined && cell ? { baseVersion: cell.version } : {}), + }; + }); +} + async function writeWithManagedLock(args: { elementId: string; value: unknown; @@ -413,12 +508,16 @@ async function writeWithManagedLock(args: { reason?: string; kind?: "set" | "create" | "delete"; artifactId?: string; + fontColor?: string; + stylePatch?: ManagedStylePatch; + preserveFontColor?: boolean; }, rt: RoomTools): Promise { - const [op] = await withBaseVersions([args], rt, args.artifactId); + const [prepared] = await prepareManagedOps([args], rt, args.artifactId); + const [op] = await withBaseVersions([prepared], rt, args.artifactId); const reason = args.reason?.trim() || `write ${args.elementId}`; - if ((args.kind ?? "set") === "set" && typeof rt.readRange === "function") { + if ((op.kind ?? "set") === "set" && typeof rt.readRange === "function") { const [current] = await rt.readRange([args.elementId], args.artifactId); - if (current && payloadsEquivalent(current.value, args.value)) { + if (current && payloadsEquivalent(current.value, op.value)) { return { ok: true, skipped: true, @@ -437,9 +536,9 @@ async function writeWithManagedLock(args: { } const lock = await rt.proposeLock([args.elementId], reason, args.artifactId); if (!lock.ok) { - if (args.kind !== "create" && args.kind !== "delete" && lock.lockId) { + if (op.kind !== "create" && op.kind !== "delete" && lock.lockId) { const draft = await rt.createDraft( - [{ elementId: args.elementId, value: args.value, baseVersion: op.baseVersion }], + [{ elementId: args.elementId, value: op.value, baseVersion: op.baseVersion }], lock.lockId, `Managed-lock draft: ${reason}`, args.artifactId, @@ -476,7 +575,7 @@ async function writeWithManagedLock(args: { let edit: EditOutcome | undefined; let release: Awaited> | undefined; try { - edit = await rt.editCell(args.elementId, args.value, op.baseVersion, args.artifactId, args.kind); + edit = await rt.editCell(args.elementId, op.value, op.baseVersion, args.artifactId, op.kind); } finally { release = await rt.releaseLock(lock.lockId); } @@ -499,7 +598,8 @@ async function writeBatchWithManagedLock(args: { reason?: string; artifactId?: string; }, rt: RoomTools): Promise> { - const ops = await withBaseVersions(args.ops, rt, args.artifactId); + const prepared = await prepareManagedOps(args.ops, rt, args.artifactId); + const ops = await withBaseVersions(prepared, rt, args.artifactId); const preflight = await unchangedSetOps({ ...args, ops }, rt); if (!preflight.activeOps.length) { const targetIds = args.ops.map((op) => op.elementId); @@ -658,6 +758,8 @@ const scalarOpInputObject = z.object({ num_fmt: z.string().nullish(), numberFormat: z.string().nullish(), number_format: z.string().nullish(), + fontColor: fontColorSchema.optional(), + font_color: fontColorSchema.optional(), text: z.any().optional(), content: z.any().optional(), expectedValue: z.any().optional(), @@ -677,7 +779,7 @@ const writeLockedCellInputSchema = scalarOpInputObject.extend({ const WRITE_LOCKED_CELL_TOOL: AgentTool = { name: "write_locked_cell", - description: "Production write path for a simple scalar cell. The runtime acquires the exact-cell lock, writes with CAS, releases in finally, and returns coordination evidence. Use this instead of propose_lock/edit_cell/release_lock when it is available.", + description: "Production write path for a scalar, formula, or style-only cell. Optional fontColor accepts RGB/ARGB hex and is stored as canonical AARRGGBB; omission preserves an existing override. The runtime acquires the exact-cell lock, writes with CAS, releases in finally, and returns coordination evidence. Use this instead of propose_lock/edit_cell/release_lock when it is available.", schema: writeLockedCellInputSchema, execute: (a: { elementId?: string; cellId?: string; value: unknown; baseVersion?: number; version?: number; reason?: string; kind?: "set" | "create" | "delete"; artifactId?: string }, rt) => writeWithManagedLock({ ...a, ...normalizeScalarOp(a), reason: a.reason, artifactId: a.artifactId }, rt), @@ -711,6 +813,8 @@ const scalarBatchSchema = z.object({ numFmt: z.any().optional(), numberFormats: z.any().optional(), numberFormat: z.any().optional(), + fontColors: z.any().optional(), + fontColor: z.any().optional(), text: z.any().optional(), content: z.any().optional(), expectedValue: z.any().optional(), @@ -729,11 +833,12 @@ const scalarBatchSchema = z.object({ const scalarParallelFields: ParallelField[] = [ { opKey: "formula", inputKeys: ["formulas", "formula"] }, { opKey: "numFmt", inputKeys: ["numFmts", "numFmt", "numberFormats", "numberFormat"] }, + { opKey: "fontColor", inputKeys: ["fontColors", "fontColor"] }, ]; const WRITE_LOCKED_CELLS_TOOL: AgentTool = { name: "write_locked_cells", - description: "Production batch write path for scalar or formula cells. Each op accepts value or formula plus an optional cached result and numFmt. The runtime preserves formula payloads, acquires one exact-range lock, writes every op with CAS, releases in finally, and returns per-cell results plus coordination evidence. Prefer this over separate lock/edit/release calls for multi-cell work.", + description: "Production batch write path for scalar, formula, or style-only cells. Each op accepts value or formula plus an optional cached result, numFmt, and canonical fontColor. Style-only writes preserve the existing payload. The runtime acquires one exact-range lock, writes every op with CAS, releases in finally, and returns per-cell results plus coordination evidence. Prefer this over separate lock/edit/release calls for multi-cell work.", schema: scalarBatchSchema, execute: (a: unknown, rt) => writeBatchWithManagedLock(normalizeBatchArgs(a, scalarParallelFields), rt), }; @@ -762,6 +867,7 @@ const WRITE_LOCKED_CELL_RESULT_TOOL: AgentTool = { confidence?: number; normalizedValue?: unknown; formula?: string; + fontColor?: string; error?: string; evidence: CellEvidence[]; reason?: string; @@ -779,6 +885,7 @@ const resultParallelFields: ParallelField[] = [ { opKey: "confidence", inputKeys: ["confidences", "confidence"] }, { opKey: "normalizedValue", inputKeys: ["normalizedValues", "normalizedValue"] }, { opKey: "formula", inputKeys: ["formulas", "formula"] }, + { opKey: "fontColor", inputKeys: ["fontColors", "fontColor"] }, { opKey: "error", inputKeys: ["errors", "error"] }, { opKey: "evidence", inputKeys: ["evidences", "evidence"] }, ]; @@ -830,6 +937,8 @@ const resultBatchSchema = z.object({ normalizedValue: z.any().optional(), formulas: z.any().optional(), formula: z.any().optional(), + fontColors: z.any().optional(), + fontColor: z.any().optional(), errors: z.any().optional(), error: z.any().optional(), evidences: z.any().optional(), @@ -871,6 +980,7 @@ const workbookOperationSchema = z.object({ formula: z.string().nullish(), result: z.any().optional(), numFmt: z.string().nullish(), + fontColor: fontColorSchema.optional(), }); const INSPECT_WORKBOOK_TOOL: AgentTool = { @@ -891,7 +1001,9 @@ const INSPECT_WORKBOOK_TOOL: AgentTool = { const readIds = [...new Set(searchIds)].slice(0, Math.max(12, Math.min(200, a.maxCells ?? 80))); const confirmed = readIds.length ? await rt.readRange(readIds, sheet) : []; const cells = mergeWorkbookCells(snapshotCells, confirmed.map((cell) => observedCell(sheet, cell.id, cell.value, cell.version))); - const inspection = inspectWorkbookTask({ instruction: a.instruction, sheetNames: [sheet], cells }); + const inspection = await verifiedWorkbookInspection(rt, a.instruction, sheet) + ?? inspectWorkbookTask({ instruction: a.instruction, sheetNames: [sheet], cells }); + const structuralRepair = await workbookStructureRepairContract(rt, a.instruction, sheet); const selected = selectWorkbookTaskCells({ inspection, cells, @@ -909,8 +1021,10 @@ const INSPECT_WORKBOOK_TOOL: AgentTool = { version: cell.version, ...(cell.formula ? { formula: cell.formula } : {}), ...(cell.numFmt ? { numFmt: cell.numFmt } : {}), + ...(cell.fontColor ? { fontColor: cell.fontColor } : {}), })), inspection, + ...(structuralRepair ? { structuralRepair } : {}), }; }, }; @@ -927,7 +1041,7 @@ const VERIFY_WORKBOOK_TOOL: AgentTool = { execute: async (a: { instruction: string; artifactId?: string; - operations: Array<{ elementId: string; baseVersion?: number; value?: unknown; formula?: string; result?: unknown; numFmt?: string }>; + operations: Array<{ elementId: string; baseVersion?: number; value?: unknown; formula?: string; result?: unknown; numFmt?: string; fontColor?: string }>; afterWrite?: boolean; }, rt) => { const snapshot = await rt.snapshot(a.artifactId); @@ -949,7 +1063,8 @@ const VERIFY_WORKBOOK_TOOL: AgentTool = { observedCellsFromSnapshot(snapshot), confirmed.map((cell) => observedCell(sheet, cell.id, cell.value, cell.version)), ); - const inspection = inspectWorkbookTask({ instruction: a.instruction, sheetNames: [sheet], cells }); + const inspection = await verifiedWorkbookInspection(rt, a.instruction, sheet) + ?? inspectWorkbookTask({ instruction: a.instruction, sheetNames: [sheet], cells }); const operations: WorkbookPlanOperation[] = a.operations.map((operation) => ({ op: "set_cell", sheet, @@ -958,6 +1073,7 @@ const VERIFY_WORKBOOK_TOOL: AgentTool = { ...(operation.formula ? { formula: operation.formula } : {}), ...(Object.prototype.hasOwnProperty.call(operation, "result") ? { result: operation.result } : {}), ...(operation.numFmt ? { numFmt: operation.numFmt } : {}), + ...(operation.fontColor ? { fontColor: normalizeSpreadsheetFontColor(operation.fontColor) } : {}), })); const plan = verifyWorkbookPlan({ instruction: a.instruction, @@ -965,6 +1081,7 @@ const VERIFY_WORKBOOK_TOOL: AgentTool = { cells, sheetNames: [sheet], operations, + afterWrite: a.afterWrite !== false, }); const candidate = a.afterWrite === false ? undefined @@ -1002,6 +1119,98 @@ const VERIFY_WORKBOOK_TOOL: AgentTool = { }, }; +type VerifiedWorkbookInspectionProvider = RoomTools & { + verifiedWorkbookInspection?: (args: { + instruction: string; + artifactId: string; + }) => WorkbookTaskInspection | undefined | Promise; +}; + +async function verifiedWorkbookInspection( + rt: RoomTools, + instruction: string, + artifactId: string, +): Promise { + const provider = rt as VerifiedWorkbookInspectionProvider; + if (typeof provider.verifiedWorkbookInspection !== "function") return undefined; + const inspection = await provider.verifiedWorkbookInspection({ instruction, artifactId }); + return inspection?.schema === 1 ? inspection : undefined; +} + +type WorkbookStructureRepairContract = { + schema: 1; + status: "complete"; + basis: "visible_workbook_invariants"; + repairId: string; + kind: string; + sheet: string; + operationCount: number; + evidence?: string[]; +}; + +type WorkbookStructureRepairProvider = RoomTools & { + workbookStructureRepairContract?: (args: { + instruction: string; + artifactId?: string; + }) => unknown | Promise; + executeWorkbookStructureRepair?: (args: { + instruction: string; + artifactId?: string; + repairId: string; + }) => unknown | Promise; +}; + +async function workbookStructureRepairContract( + rt: RoomTools, + instruction: string, + artifactId?: string, +): Promise { + const provider = rt as WorkbookStructureRepairProvider; + if (typeof provider.workbookStructureRepairContract !== "function") return undefined; + const value = await provider.workbookStructureRepairContract({ instruction, artifactId }); + const record = workbookRecord(value); + return record?.schema === 1 + && record.status === "complete" + && record.basis === "visible_workbook_invariants" + && typeof record.repairId === "string" + && typeof record.kind === "string" + && typeof record.sheet === "string" + && typeof record.operationCount === "number" + ? value as WorkbookStructureRepairContract + : undefined; +} + +export const EXECUTE_WORKBOOK_STRUCTURE_REPAIR_TOOL_NAME = "execute_workbook_structure_repair" as const; + +export const EXECUTE_WORKBOOK_STRUCTURE_REPAIR_TOOL: AgentTool = { + name: EXECUTE_WORKBOOK_STRUCTURE_REPAIR_TOOL_NAME, + description: "Execute a complete, agent-visible structural workbook repair contract after inspection. This is capability-gated and only accepts a repairId issued by the current room; it cannot invent row operations or bypass workbook verification.", + schema: z.object({ + instruction: z.string().min(1).describe("the complete workbook task being repaired"), + artifactId: z.string().optional().describe("the workbook sheet containing the structural defect"), + repairId: z.string().min(1).describe("the exact visible-invariant repairId returned by inspect_workbook"), + }), + execute: async (a: { instruction: string; artifactId?: string; repairId: string }, rt) => { + const provider = rt as WorkbookStructureRepairProvider; + if (typeof provider.executeWorkbookStructureRepair !== "function") { + return { + ok: false, + status: "unsupported", + error: "This room does not expose governed structural workbook mutations.", + }; + } + const contract = await workbookStructureRepairContract(rt, a.instruction, a.artifactId); + if (!contract || contract.repairId !== a.repairId) { + return { + ok: false, + status: "needs_repair", + error: "The structural repair contract is missing, stale, or does not match this workbook.", + }; + } + return provider.executeWorkbookStructureRepair(a); + }, +}; + export const EXECUTE_VERIFIED_WORKBOOK_PLAN_TOOL: AgentTool = { name: "execute_verified_workbook_plan", description: "Execute the complete high-confidence plan already identified by inspect_workbook without echoing every formula through the model. This tool re-inspects visible workbook evidence, refuses ambiguous or incomplete plans, preflights the exact operations, writes through managed locks and CAS, and post-verifies every target. Use it after inspect_workbook when that inspection returns exact formulaFillSuggestions, formulaRepairSuggestions, or valueSuggestions for a multi-cell task.", @@ -1127,79 +1336,131 @@ export const EXECUTE_VERIFIED_WORKBOOK_PLAN_TOOL: AgentTool = { }; } - const preflightResult = await VERIFY_WORKBOOK_TOOL.execute({ - instruction: a.instruction, - artifactId, - operations: fullOperations, - afterWrite: false, - }, rt); - const preflight = workbookVerificationSummary(preflightResult); - if (preflight.status !== "passed") { - return { - ok: false, - status: "needs_repair", + // Filename-led audits deliberately cap each independently verified edit set + // at eight cells. A complete structural contract may be larger, so retain + // that guard by preflighting and post-verifying deterministic chunks rather + // than weakening the audit bound or asking the model to truncate the plan. + const maxBatchSize = inspection.auditFocus ? 8 : fullOperations.length; + const fullOperationBatches = chunkWorkbookOperations(fullOperations, maxBatchSize); + const changedByElementId = new Map(operations.map((operation) => [operation.elementId, operation])); + const preflightSummaries: Record[] = []; + for (const batch of fullOperationBatches) { + const result = await VERIFY_WORKBOOK_TOOL.execute({ + instruction: a.instruction, artifactId, - operationCount: fullOperations.length, - changedTargetCount: 0, - error: "workbook_preflight_rejected", - repairPrompt: workbookRecord(preflightResult)?.repairPrompt, - phases: { - inspect: workbookInspectSummary(inspectResult), - plan: workbookPlanSummary(fullOperations), - preflight, - write: { status: "skipped" }, - verify: { status: "skipped" }, - }, - }; + operations: batch, + afterWrite: false, + }, rt); + const summary = workbookVerificationSummary(result); + preflightSummaries.push(summary); + if (summary.status !== "passed") { + return { + ok: false, + status: "needs_repair", + artifactId, + operationCount: fullOperations.length, + changedTargetCount: 0, + error: "workbook_preflight_rejected", + repairPrompt: workbookRecord(result)?.repairPrompt, + phases: { + inspect: workbookInspectSummary(inspectResult), + plan: workbookPlanSummary(fullOperations), + preflight: aggregateWorkbookVerificationSummaries(preflightSummaries, fullOperationBatches.length), + write: { status: "skipped" }, + verify: { status: "skipped" }, + }, + }; + } } - const writeResult = await WRITE_LOCKED_CELLS_TOOL.execute({ - artifactId, - reason: a.reason?.trim() || "execute verified workbook plan", - ops: operations, - }, rt); - const write = workbookWriteSummary(writeResult, operations); - if (write.status !== "completed") { - const pendingApproval = write.status === "pending_approval"; - return { - ok: false, - status: pendingApproval ? "pending_approval" : "blocked", + const writeSummaries: Array & { status: string; committedCount: number }> = []; + const verificationResults: unknown[] = []; + const verificationSummaries: Record[] = []; + let changedTargetCount = 0; + for (const batch of fullOperationBatches) { + const changedBatch = batch.flatMap((operation) => { + const changed = changedByElementId.get(operation.elementId); + return changed ? [changed] : []; + }); + if (changedBatch.length > 0) { + const writeResult = await WRITE_LOCKED_CELLS_TOOL.execute({ + artifactId, + reason: a.reason?.trim() || "execute verified workbook plan", + ops: changedBatch, + }, rt); + const write = workbookWriteSummary(writeResult, changedBatch); + writeSummaries.push(write); + changedTargetCount += write.committedCount; + if (write.status !== "completed") { + const pendingApproval = write.status === "pending_approval"; + return { + ok: false, + status: pendingApproval ? "pending_approval" : "blocked", + artifactId, + operationCount: fullOperations.length, + changedTargetCount, + error: pendingApproval ? "workbook_plan_pending_approval" : "managed_workbook_write_blocked", + repairPrompt: pendingApproval + ? "Wait for the host to approve or reject the proposed workbook edits; do not report them as committed." + : "Resolve the managed-lock, draft, or CAS state before post-write verification.", + phases: { + inspect: workbookInspectSummary(inspectResult), + plan: workbookPlanSummary(fullOperations), + preflight: aggregateWorkbookVerificationSummaries(preflightSummaries, fullOperationBatches.length), + write: aggregateWorkbookWriteSummaries(writeSummaries, operations.length, fullOperationBatches.length), + verify: aggregateWorkbookVerificationSummaries(verificationSummaries, fullOperationBatches.length), + }, + }; + } + } + + const verificationResult = await VERIFY_WORKBOOK_TOOL.execute({ + instruction: a.instruction, artifactId, - operationCount: fullOperations.length, - changedTargetCount: write.committedCount, - error: pendingApproval ? "workbook_plan_pending_approval" : "managed_workbook_write_blocked", - repairPrompt: pendingApproval - ? "Wait for the host to approve or reject the proposed workbook edits; do not report them as committed." - : "Resolve the managed-lock, draft, or CAS state before post-write verification.", - phases: { - inspect: workbookInspectSummary(inspectResult), - plan: workbookPlanSummary(fullOperations), - preflight, - write, - verify: { status: "skipped" }, - }, - }; + operations: batch, + afterWrite: true, + }, rt); + const verify = workbookVerificationSummary(verificationResult); + verificationResults.push(verificationResult); + verificationSummaries.push(verify); + if (verify.status !== "passed") { + return { + ok: false, + status: "needs_repair", + artifactId, + operationCount: fullOperations.length, + changedTargetCount, + error: "workbook_post_write_verification_failed", + repairPrompt: workbookRecord(verificationResult)?.repairPrompt, + phases: { + inspect: workbookInspectSummary(inspectResult), + plan: workbookPlanSummary(fullOperations), + preflight: aggregateWorkbookVerificationSummaries(preflightSummaries, fullOperationBatches.length), + write: aggregateWorkbookWriteSummaries(writeSummaries, operations.length, fullOperationBatches.length), + verify: aggregateWorkbookVerificationSummaries(verificationSummaries, fullOperationBatches.length), + }, + }; + } } - const verificationResult = await VERIFY_WORKBOOK_TOOL.execute({ - instruction: a.instruction, - artifactId, - operations: fullOperations, - afterWrite: true, - }, rt); - const verify = workbookVerificationSummary(verificationResult); - const completed = verify.status === "passed"; + const preflight = aggregateWorkbookVerificationSummaries(preflightSummaries, fullOperationBatches.length); + const write = aggregateWorkbookWriteSummaries(writeSummaries, operations.length, fullOperationBatches.length); + const verify = aggregateWorkbookVerificationSummaries(verificationSummaries, fullOperationBatches.length); + const completed = verificationSummaries.length === fullOperationBatches.length + && verificationSummaries.every((summary) => summary.status === "passed"); return { ok: completed, status: completed ? "completed" : "needs_repair", artifactId, operationCount: fullOperations.length, - changedTargetCount: write.committedCount, + changedTargetCount, approvalBoundary: "RoomTools managed lock and compare-and-set", workflowComplete: completed, ...(completed ? { nextAction: "Return the final answer. Do not call another workbook tool." } : { error: "workbook_post_write_verification_failed", - repairPrompt: workbookRecord(verificationResult)?.repairPrompt, + repairPrompt: verificationResults + .map((result) => workbookRecord(result)?.repairPrompt) + .find((prompt) => typeof prompt === "string"), }), phases: { inspect: workbookInspectSummary(inspectResult), @@ -1212,6 +1473,52 @@ export const EXECUTE_VERIFIED_WORKBOOK_PLAN_TOOL: AgentTool = { }, }; +function chunkWorkbookOperations(operations: T[], maxBatchSize: number): T[][] { + const size = Math.max(1, Math.trunc(maxBatchSize)); + const batches: T[][] = []; + for (let index = 0; index < operations.length; index += size) { + batches.push(operations.slice(index, index + size)); + } + return batches; +} + +function aggregateWorkbookVerificationSummaries( + summaries: Record[], + expectedBatchCount: number, +): Record { + const passed = summaries.length === expectedBatchCount && summaries.every((summary) => summary.status === "passed"); + return { + status: passed ? "passed" : "needs_repair", + receiptHash: stableTraceHash(summaries), + issueCount: summaries.reduce((total, summary) => total + Number(summary.issueCount ?? 0), 0), + checkedCount: summaries.reduce((total, summary) => total + Number(summary.checkedCount ?? 0), 0), + batchCount: summaries.length, + expectedBatchCount, + batches: summaries, + }; +} + +function aggregateWorkbookWriteSummaries( + summaries: Array & { status: string; committedCount: number }>, + targetCount: number, + expectedBatchCount: number, +): Record & { status: string; committedCount: number } { + const blocked = summaries.some((summary) => summary.status === "blocked"); + const pendingApproval = summaries.some((summary) => summary.status === "pending_approval"); + const committedCount = summaries.reduce((total, summary) => total + summary.committedCount, 0); + return { + status: blocked ? "blocked" : pendingApproval ? "pending_approval" : "completed", + receiptHash: stableTraceHash(summaries), + targetCount, + committedCount, + skippedCount: summaries.reduce((total, summary) => total + Number(summary.skippedCount ?? 0), 0), + failedCount: summaries.reduce((total, summary) => total + Number(summary.failedCount ?? 0), 0), + batchCount: summaries.length, + expectedBatchCount, + batches: summaries, + }; +} + function workbookRecord(value: unknown): Record | undefined { return value && typeof value === "object" && !Array.isArray(value) ? value as Record : undefined; } @@ -1283,14 +1590,19 @@ function workbookWriteSummary( function workbookOperationAlreadySatisfied(operation: WorkbookSuggestedPlanOperation, rawValue: unknown): boolean { const record = workbookRecord(rawValue); const formula = typeof record?.formula === "string" ? record.formula : undefined; + const fontColorMatches = !operation.fontColor + || normalizeSpreadsheetFontColor(record?.fontColor) === normalizeSpreadsheetFontColor(operation.fontColor); if (operation.formula) { return normalizeWorkbookFormula(formula) === normalizeWorkbookFormula(operation.formula) - && (!operation.numFmt || record?.numFmt === operation.numFmt); + && (!operation.numFmt || record?.numFmt === operation.numFmt) + && fontColorMatches; } const scalar = record && Object.prototype.hasOwnProperty.call(record, "value") ? record.value : rawValue; - return Object.prototype.hasOwnProperty.call(operation, "value") - && payloadsEquivalent(scalar, operation.value) - && (!operation.numFmt || record?.numFmt === operation.numFmt); + const valueMatches = !Object.prototype.hasOwnProperty.call(operation, "value") + || payloadsEquivalent(scalar, operation.value); + return valueMatches + && (!operation.numFmt || record?.numFmt === operation.numFmt) + && fontColorMatches; } function normalizeWorkbookFormula(value: string | undefined): string | undefined { @@ -1314,12 +1626,14 @@ function observedCell(sheet: string, address: string, rawValue: unknown, version const payload = record && "value" in record ? record : undefined; const formula = typeof record?.formula === "string" ? record.formula : undefined; const numFmt = typeof record?.numFmt === "string" ? record.numFmt : undefined; + const fontColor = normalizeSpreadsheetFontColor(record?.fontColor); return { sheet, address, value: payload ? payload.value : rawValue, ...(formula ? { formula } : {}), ...(numFmt ? { numFmt } : {}), + ...(fontColor ? { fontColor } : {}), ...(version === undefined ? {} : { version }), }; } @@ -1336,6 +1650,7 @@ function mergeWorkbookCells(...groups: WorkbookObservedCell[][]): WorkbookObserv export const ROOM_TOOLS: AgentTool[] = [ INSPECT_WORKBOOK_TOOL, VERIFY_WORKBOOK_TOOL, + EXECUTE_WORKBOOK_STRUCTURE_REPAIR_TOOL, { name: "read_range", description: "Read the current value + version of specific cells. Works even on LOCKED cells (locked = read-only, not invisible). Call this before editing, and again after any conflict. Defaults to the primary file ONLY. For uploaded source workbooks or any non-primary file, you MUST pass artifactId from list_artifacts; A1-style ids like A1/B2 without artifactId usually read the blank Sheet 1 and are wrong. If you omit elementIds, the tool returns a bounded artifact sample and instructions instead of dumping the file.", diff --git a/src/nodeagent/skills/spreadsheet/workbookTaskIntelligence.ts b/src/nodeagent/skills/spreadsheet/workbookTaskIntelligence.ts index 9c7860a7..0df8cf3f 100644 --- a/src/nodeagent/skills/spreadsheet/workbookTaskIntelligence.ts +++ b/src/nodeagent/skills/spreadsheet/workbookTaskIntelligence.ts @@ -1,12 +1,27 @@ +import { normalizeSpreadsheetFontColor } from "../../../shared/spreadsheetFontColor"; + export type WorkbookObservedCell = { sheet: string; address: string; value: unknown; formula?: string; numFmt?: string; + fontColor?: string; version?: number; }; +export type WorkbookAuditFocus = + | "incorrect_average" + | "embedded_hardcode" + | "color_coding" + | "formula_errors" + | "double_counting" + | "index_match" + | "cross_sheet_reference" + | "unit_mismatch" + | "sign_convention" + | "relative_absolute_reference"; + export type WorkbookReferenceRole = "target" | "dependency" | "ambiguous"; export type WorkbookTaskReference = { @@ -29,6 +44,7 @@ export type WorkbookInspectionFindingKind = | "named_year_target_band" | "semantic_formula_target" | "formula_range_anomaly" + | "font_color_anomaly" | "lookup_bounds_missing" | "implicit_assignment_target"; @@ -60,8 +76,20 @@ export type WorkbookBlockedTarget = { export type WorkbookTaskInspection = { schema: 1; + auditFocus?: { + kind: WorkbookAuditFocus; + source: "agent_visible_filename"; + workbookName: string; + }; mutatingTask: boolean; allowEmptyPlan: boolean; + deterministicPlan?: { + status: "complete"; + basis: "visible_workbook_invariants"; + auditFocus: Exclude; + operationCount: number; + sheets: string[]; + }; referencedSheets: string[]; explicitReferences: WorkbookTaskReference[]; targetCandidates: Array<{ sheet: string; address: string; reason: string }>; @@ -92,6 +120,14 @@ export type WorkbookTaskInspection = { numFmt?: string; evidence: string[]; }>; + styleSuggestions: Array<{ + kind: "font_color"; + confidence: "high"; + sheet: string; + cell: string; + fontColor: string; + evidence: string[]; + }>; rankedCellKeys: string[]; recommendedReads: Array<{ sheet: string; addresses: string[]; reason: string }>; completionChecks: string[]; @@ -105,6 +141,7 @@ export type WorkbookPlanOperation = { formula?: string; result?: unknown; numFmt?: string; + fontColor?: string; [key: string]: unknown; }; @@ -113,6 +150,7 @@ export type WorkbookSuggestedPlanOperation = { formula?: string; value?: string | number | boolean; numFmt?: string; + fontColor?: string; }; export type WorkbookSuggestedPlan = { @@ -134,9 +172,13 @@ export type WorkbookPlanIssueKind = | "formula_self_reference" | "formula_semantic_mismatch" | "value_semantic_mismatch" + | "font_color_semantic_mismatch" | "unsafe_lookup_bounds" | "malformed_formula" - | "duplicate_target"; + | "duplicate_target" + | "overbroad_audit_plan" + | "unsubstantiated_audit_target" + | "audit_style_content_overwrite"; export type WorkbookPlanIssue = { kind: WorkbookPlanIssueKind; @@ -167,6 +209,7 @@ export type WorkbookValueCheck = { expectedValue?: unknown; expectedFormula?: string; expectedNumFmt?: string; + expectedFontColor?: string; allowBlank?: boolean; }; @@ -175,6 +218,7 @@ export type WorkbookValueCheckResult = WorkbookValueCheck & { actualValue: unknown; actualFormula?: string; actualNumFmt?: string; + actualFontColor?: string; version?: number; issues: string[]; }; @@ -240,7 +284,29 @@ export function normalizeAddress(address: string): string { } export function normalizeFormula(formula: string | undefined): string | undefined { - const normalized = formula?.trim().replace(/^=/, "").replace(/\s+/g, ""); + const source = formula?.trim().replace(/^=/, "") ?? ""; + let normalized = ""; + let quote: "'" | '"' | undefined; + for (let index = 0; index < source.length; index += 1) { + const character = source[index]; + if (quote) { + normalized += character; + if (character !== quote) continue; + if (source[index + 1] === quote) { + normalized += source[index + 1]; + index += 1; + } else { + quote = undefined; + } + continue; + } + if (character === "'" || character === '"') { + quote = character; + normalized += character; + } else if (!/\s/.test(character)) { + normalized += character; + } + } return normalized || undefined; } @@ -264,6 +330,7 @@ export function buildWorkbookSuggestedPlan( ...(operation.formula ? { formula: operation.formula.trim().replace(/^=/, "") } : {}), ...(Object.prototype.hasOwnProperty.call(operation, "value") ? { value: operation.value } : {}), ...(operation.numFmt ? { numFmt: operation.numFmt } : {}), + ...(operation.fontColor ? { fontColor: normalizeSpreadsheetFontColor(operation.fontColor) } : {}), }; const list = candidates.get(elementId) ?? []; if (!list.some((candidate) => suggestedOperationsEquivalent(candidate, normalized))) list.push(normalized); @@ -290,6 +357,11 @@ export function buildWorkbookSuggestedPlan( ...(suggestion.numFmt ? { numFmt: suggestion.numFmt } : {}), }); } + for (const suggestion of inspection.styleSuggestions) { + if (suggestion.confidence !== "high" || suggestion.sheet.trim().toLowerCase() !== targetSheet) continue; + if (!targetKeys.has(workbookCellKey(suggestion.sheet, suggestion.cell))) continue; + add({ elementId: suggestion.cell, fontColor: suggestion.fontColor }); + } const conflicts = [...candidates.entries()] .filter(([, values]) => values.length > 1) @@ -308,30 +380,36 @@ function suggestedOperationsEquivalent( ): boolean { return normalizeFormula(left.formula) === normalizeFormula(right.formula) && valuesEquivalent(left.value, right.value) - && (left.numFmt ?? "") === (right.numFmt ?? ""); + && (left.numFmt ?? "") === (right.numFmt ?? "") + && (left.fontColor ?? "") === (right.fontColor ?? ""); } export function extractWorkbookTaskReferences(instruction: string, sheetNames: string[] = []): WorkbookTaskReference[] { const references: WorkbookTaskReference[] = []; const occupied = new Set(); const sheetByLower = new Map(sheetNames.map((sheet) => [sheet.toLowerCase(), sheet])); - const sheetPattern = /(?:'([^']+)'|([A-Za-z0-9_. -]+))!\s*(\$?[A-Z]{1,3}\$?[1-9][0-9]*)(?:\s*(?::|-|\bto\b)\s*(\$?[A-Z]{1,3}\$?[1-9][0-9]*))?/gi; + const sheetPattern = /(?:'((?:[^']|'')+)'|([A-Za-z_][A-Za-z0-9_.]*))!\s*(\$?[A-Z]{1,3}\$?[1-9][0-9]*)(?:\s*(?::|-|\bto\b)\s*(\$?[A-Z]{1,3}\$?[1-9][0-9]*))?/gi; for (const match of instruction.matchAll(sheetPattern)) { const sourceText = match[0]; + const index = match.index ?? 0; + if (!hasA1ReferenceBoundaries(instruction, index, sourceText.length)) continue; const rawSheet = (match[1] ?? match[2] ?? "").trim(); - const sheet = sheetByLower.get(rawSheet.toLowerCase()) ?? rawSheet; + const unescapedSheet = rawSheet.replace(/''/g, "'"); + const sheet = sheetByLower.get(unescapedSheet.toLowerCase()) ?? unescapedSheet; const start = normalizeAddress(match[3]); const end = normalizeAddress(match[4] ?? match[3]); - references.push({ sheet, start, end, sourceText, role: referenceRole(instruction, match.index ?? 0, sourceText.length) }); - for (let index = match.index ?? 0; index < (match.index ?? 0) + sourceText.length; index += 1) occupied.add(String(index)); + references.push({ sheet, start, end, sourceText, role: referenceRole(instruction, index, sourceText.length) }); + for (let offset = index; offset < index + sourceText.length; offset += 1) occupied.add(String(offset)); } - const rangePattern = /(\$?[A-Z]{1,3}\$?[1-9][0-9]*)(?:\s*(?::|-|\bto\b)\s*(\$?[A-Z]{1,3}\$?[1-9][0-9]*))?/gi; + const rangePattern = /(\$?[A-Z]{1,3}\$?[1-9][0-9]*)(?:\s*(:|-|\bto\b)\s*(\$?[A-Z]{1,3}\$?[1-9][0-9]*))?/gi; for (const match of instruction.matchAll(rangePattern)) { const index = match.index ?? 0; if (occupied.has(String(index))) continue; + if (!hasA1ReferenceBoundaries(instruction, index, match[0].length)) continue; const start = normalizeAddress(match[1]); - const end = normalizeAddress(match[2] ?? match[1]); + const end = normalizeAddress(match[3] ?? match[1]); + if (!shouldExtractUnqualifiedA1Reference(instruction, index, match[0].length, start, end, match[2])) continue; references.push({ start, end, sourceText: match[0], role: referenceRole(instruction, index, match[0].length) }); } @@ -349,6 +427,85 @@ export function extractWorkbookTaskReferences(instruction: string, sheetNames: s return [...unique.values()]; } +function hasA1ReferenceBoundaries(instruction: string, index: number, length: number): boolean { + const before = instruction[index - 1] ?? ""; + const after = instruction[index + length] ?? ""; + return !/[A-Za-z0-9_]/.test(before) && !/[A-Za-z0-9_]/.test(after); +} + +function shouldExtractUnqualifiedA1Reference( + instruction: string, + index: number, + length: number, + start: string, + end: string, + separator: string | undefined, +): boolean { + const clause = referenceClause(instruction, index, length); + const explicitCellContext = /\b(?:cells?|ranges?|addresses?|formulas?)\b/i.test(clause); + const formulaContext = hasFormulaReferenceContext(instruction, index, length); + if (hasOpaqueIdLabel(instruction, index, length)) return false; + + if (start !== end) { + const normalizedSeparator = separator?.trim().toLowerCase(); + return normalizedSeparator === ":" || normalizedSeparator === "to" || explicitCellContext || formulaContext; + } + if (/^Q[1-4]$/i.test(start)) { + const local = instruction.slice(Math.max(0, index - 24), Math.min(instruction.length, index + length + 24)); + const quoted = start.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const explicitQuarterCell = new RegExp( + `(?:\\b(?:cell|address)\\s+['\"]?${quoted}\\b|\\b${quoted}['\"]?\\s+(?:cell|address)\\b)`, + "i", + ).test(local); + return explicitQuarterCell || formulaContext; + } + if (explicitCellContext || formulaContext) return true; + + const immediate = instruction.slice(Math.max(0, index - 70), Math.min(instruction.length, index + length + 70)); + return TARGET_CONTEXT_RE.test(immediate) || DEPENDENCY_CONTEXT_RE.test(immediate); +} + +function referenceClause(instruction: string, index: number, length: number): string { + const before = instruction.slice(0, index); + const start = Math.max( + before.lastIndexOf("."), + before.lastIndexOf(";"), + before.lastIndexOf("?"), + before.lastIndexOf("!"), + before.lastIndexOf("\n"), + ) + 1; + const after = instruction.slice(index + length); + const nextBoundary = after.search(/[.;!?\n]/); + const end = nextBoundary === -1 ? instruction.length : index + length + nextBoundary; + return instruction.slice(start, end); +} + +function hasFormulaReferenceContext(instruction: string, index: number, length: number): boolean { + const before = instruction.slice(Math.max(0, index - 180), index); + const after = instruction.slice(index + length, Math.min(instruction.length, index + length + 12)); + const clauseStart = Math.max( + before.lastIndexOf("."), + before.lastIndexOf(";"), + before.lastIndexOf("?"), + before.lastIndexOf("!"), + before.lastIndexOf("\n"), + ) + 1; + const formulaPrefix = before.slice(clauseStart); + if (/=[^=;!?\n]*$/.test(formulaPrefix)) return true; + if (/\b[A-Z][A-Z0-9_.]*\([^()]*$/i.test(formulaPrefix)) return true; + const previous = before.match(/\S\s*$/)?.[0].trim() ?? ""; + const next = after.match(/^\s*(\S)/)?.[1] ?? ""; + return (!!previous && "=+-*/^".includes(previous)) || (!!next && "+-*/^<>=".includes(next)); +} + +function hasOpaqueIdLabel(instruction: string, index: number, length: number): boolean { + const before = instruction.slice(Math.max(0, index - 60), index); + const after = instruction.slice(index + length, Math.min(instruction.length, index + length + 60)); + const idLabel = "(?:artifact(?:\\s*id)?|artifactid|room(?:\\s*id)?|roomid|job(?:\\s*id)?|jobid|trace(?:\\s*id)?|traceid|request(?:\\s*id)?|requestid)"; + return new RegExp(`\\b${idLabel}\\s*[:=#-]?\\s*$`, "i").test(before) + || new RegExp(`^\\s*${idLabel}\\b`, "i").test(after); +} + export function inspectWorkbookTask(args: { instruction: string; sheetNames: string[]; @@ -356,6 +513,7 @@ export function inspectWorkbookTask(args: { maxFindings?: number; }): WorkbookTaskInspection { const maxFindings = Math.max(1, Math.min(args.maxFindings ?? 24, 100)); + const auditFocus = workbookAuditFocus(args.instruction); const explicitReferences = extractWorkbookTaskReferences(args.instruction, args.sheetNames); const referencedSheets = args.sheetNames.filter((sheet) => new RegExp(`(?:^|[^a-z0-9])${escapeRegExp(sheet)}(?:$|[^a-z0-9])`, "i").test(args.instruction)); const cellsByKey = new Map(args.cells.map((cell) => [workbookCellKey(cell.sheet, cell.address), cell])); @@ -364,6 +522,7 @@ export function inspectWorkbookTask(args: { const formulaFillSuggestions: WorkbookTaskInspection["formulaFillSuggestions"] = []; const formulaRepairSuggestions: WorkbookTaskInspection["formulaRepairSuggestions"] = []; const valueSuggestions: WorkbookTaskInspection["valueSuggestions"] = []; + const styleSuggestions: WorkbookTaskInspection["styleSuggestions"] = []; const blockedTargets: WorkbookBlockedTarget[] = []; const targetCandidates = new Map(); const targetBands = new Map(); @@ -552,35 +711,99 @@ export function inspectWorkbookTask(args: { } const formulaBandAnalysis = analyzeFormulaBands(args.cells, addRank); + const compatibleFormulaSuggestions = (auditFocus + ? formulaBandAnalysis.suggestions.filter((suggestion) => auditFocusAllowsFormulaSuggestion( + auditFocus.kind, + suggestion, + cellsByKey.get(workbookCellKey(suggestion.sheet, suggestion.cell)), + )).slice(0, 8) + : formulaBandAnalysis.suggestions); + const compatibleFormulaKeys = new Set(compatibleFormulaSuggestions + .map((suggestion) => workbookCellKey(suggestion.sheet, suggestion.cell))); findings.push(...formulaBandAnalysis.findings.filter((finding) => - !weekdayTargetKeys.has(workbookCellKey(finding.sheet, finding.address)))); + !weekdayTargetKeys.has(workbookCellKey(finding.sheet, finding.address)) + && (!auditFocus || compatibleFormulaKeys.has(workbookCellKey(finding.sheet, finding.address))))); const requireFormulaAnomalyRepairs = genericFormulaAuditTask(args.instruction); - for (const suggestion of formulaBandAnalysis.suggestions) { + for (const suggestion of compatibleFormulaSuggestions) { if (weekdayTargetKeys.has(workbookCellKey(suggestion.sheet, suggestion.cell))) continue; formulaRepairSuggestions.push(suggestion); - if (requireFormulaAnomalyRepairs) { + if (requireFormulaAnomalyRepairs || auditFocus) { addCandidate("target", cellsByKey.get(workbookCellKey(suggestion.sheet, suggestion.cell)) ?? { sheet: suggestion.sheet, address: suggestion.cell, value: "", - }, `two-sided formula pattern agrees on ${suggestion.formula}`); + }, auditFocus + ? `${auditFocus.workbookName} identifies the audit class and two-sided workbook evidence agrees on ${suggestion.formula}` + : `two-sided formula pattern agrees on ${suggestion.formula}`); } } - const averageRangeAnalysis = analyzeAverageFormulaRanges(args.instruction, args.cells, addRank); - findings.push(...averageRangeAnalysis.findings); - for (const suggestion of averageRangeAnalysis.suggestions) { + const averageRangeAnalysis = !auditFocus || auditFocus.kind === "incorrect_average" + ? analyzeAverageFormulaRanges(args.instruction, args.cells, addRank) + : { findings: [], suggestions: [] as WorkbookTaskInspection["formulaRepairSuggestions"] }; + const supportedAverageKeys = new Map(compatibleFormulaSuggestions + .filter((suggestion) => auditFocus?.kind !== "incorrect_average" || suggestion.kind === "replace_outlier") + .map((suggestion) => [workbookCellKey(suggestion.sheet, suggestion.cell), normalizeFormula(suggestion.formula)])); + const averageSuggestions = auditFocus?.kind === "incorrect_average" + ? averageRangeAnalysis.suggestions.filter((suggestion) => + supportedAverageKeys.get(workbookCellKey(suggestion.sheet, suggestion.cell)) === normalizeFormula(suggestion.formula) + || suggestion.evidence.some((item) => item.startsWith("locally confirmed contiguous expansion:"))) + : averageRangeAnalysis.suggestions; + const averageSuggestionKeys = new Set(averageSuggestions.map((suggestion) => workbookCellKey(suggestion.sheet, suggestion.cell))); + findings.push(...averageRangeAnalysis.findings.filter((finding) => + !auditFocus || averageSuggestionKeys.has(workbookCellKey(finding.sheet, finding.address)))); + for (const suggestion of averageSuggestions) { const key = workbookCellKey(suggestion.sheet, suggestion.cell); for (let index = formulaRepairSuggestions.length - 1; index >= 0; index -= 1) { const current = formulaRepairSuggestions[index]; if (workbookCellKey(current.sheet, current.cell) === key) formulaRepairSuggestions.splice(index, 1); } formulaRepairSuggestions.push(suggestion); - addCandidate("target", cellsByKey.get(workbookCellKey(suggestion.sheet, suggestion.cell)) ?? { - sheet: suggestion.sheet, - address: suggestion.cell, - value: "", - }, suggestion.evidence.join("; ")); + if (!auditFocus) { + addCandidate("target", cellsByKey.get(workbookCellKey(suggestion.sheet, suggestion.cell)) ?? { + sheet: suggestion.sheet, + address: suggestion.cell, + value: "", + }, suggestion.evidence.join("; ")); + } + } + + const focusedAuditAnalysis = auditFocus + ? analyzeFocusedAuditPatterns(auditFocus.kind, args.cells) + : { + findings: [] as WorkbookInspectionFinding[], + formulaSuggestions: [] as WorkbookTaskInspection["formulaRepairSuggestions"], + valueSuggestions: [] as WorkbookTaskInspection["valueSuggestions"], + }; + const focusedFormulaKeys = new Set(); + const focusedValueKeys = new Set(); + findings.push(...focusedAuditAnalysis.findings); + for (const suggestion of focusedAuditAnalysis.formulaSuggestions) { + const key = workbookCellKey(suggestion.sheet, suggestion.cell); + focusedFormulaKeys.add(key); + formulaRepairSuggestions.push(suggestion); + const target = cellsByKey.get(key) ?? { sheet: suggestion.sheet, address: suggestion.cell, value: "" }; + addRank(target, 248, "focused_audit_formula_anomaly"); + addCandidate("target", target, suggestion.evidence.join("; ")); + } + for (const suggestion of focusedAuditAnalysis.valueSuggestions) { + const key = workbookCellKey(suggestion.sheet, suggestion.cell); + focusedValueKeys.add(key); + valueSuggestions.push(suggestion); + const target = cellsByKey.get(key) ?? { sheet: suggestion.sheet, address: suggestion.cell, value: "" }; + addRank(target, 248, "focused_audit_value_anomaly"); + addCandidate("target", target, suggestion.evidence.join("; ")); + } + + if (auditFocus?.kind === "color_coding") { + const colorAnalysis = analyzeFontColorAudit(args.cells); + findings.push(...colorAnalysis.findings); + for (const suggestion of colorAnalysis.suggestions) { + styleSuggestions.push(suggestion); + const target = cellsByKey.get(workbookCellKey(suggestion.sheet, suggestion.cell)); + addRank(target, 245, "font_color_anomaly"); + addCandidate("target", target, suggestion.evidence.join("; ")); + } } if (FORMULA_TASK_RE.test(args.instruction)) { @@ -847,28 +1070,89 @@ export function inspectWorkbookTask(args: { if (last) addRank(last, 20, "sheet_extent"); } + const explicitTargetKeys = new Set([...targetBands.values()] + .filter((band) => band.source === "explicit_reference") + .flatMap((band) => band.addresses.map((address) => workbookCellKey(band.sheet, address)))); + const focusTargetKeys = new Set([ + ...explicitTargetKeys, + ...compatibleFormulaKeys, + ...averageSuggestionKeys, + ...focusedFormulaKeys, + ...focusedValueKeys, + ...styleSuggestions.map((suggestion) => workbookCellKey(suggestion.sheet, suggestion.cell)), + ]); + if (auditFocus?.kind === "formula_errors") { + for (const finding of findings.filter((candidate) => candidate.kind === "formula_error" || candidate.kind === "formula_self_reference")) { + const key = workbookCellKey(finding.sheet, finding.address); + focusTargetKeys.add(key); + addCandidate("target", cellsByKey.get(key), `visible workbook error at ${finding.sheet}!${finding.address}`); + } + } const rankedCells = [...ranked.values()].sort((left, right) => right.score - left.score || left.sheet.localeCompare(right.sheet) || compareAddresses(left.address, right.address)); - const boundedFindings = dedupeFindings(findings).slice(0, maxFindings); + const boundedFindings = dedupeFindings(findings) + .filter((finding) => !auditFocus || auditFocusAllowsFinding(auditFocus.kind, finding, focusTargetKeys)) + .slice(0, maxFindings); const recommendedReads = recommendedReadGroups(rankedCells.slice(0, 40)); const allowEmptyPlan = EMPTY_PLAN_ALLOWED_RE.test(args.instruction); const mutatingTask = (MUTATING_TASK_RE.test(args.instruction) || METHOD_MUTATING_TASK_RE.test(args.instruction) || IMPLICIT_ASSIGNMENT_RE.test(args.instruction)) && !/\b(?:explain|describe|summari[sz]e)\s+only\b/i.test(args.instruction); + const dedupedFormulaRepairs = dedupeFormulaRepairSuggestions(formulaRepairSuggestions).slice(0, 64); + const focusedValueSuggestions = auditFocus + ? valueSuggestions.filter((suggestion) => focusedValueKeys.has(workbookCellKey(suggestion.sheet, suggestion.cell))) + : valueSuggestions; + const deterministicAuditKinds = new Set([ + "incorrect_average", + "embedded_hardcode", + "color_coding", + "double_counting", + "index_match", + "cross_sheet_reference", + "unit_mismatch", + "sign_convention", + "relative_absolute_reference", + ]); + const deterministicOperations = [ + ...dedupedFormulaRepairs.map((suggestion) => ({ sheet: suggestion.sheet, cell: suggestion.cell })), + ...focusedValueSuggestions.map((suggestion) => ({ sheet: suggestion.sheet, cell: suggestion.cell })), + ...styleSuggestions.map((suggestion) => ({ sheet: suggestion.sheet, cell: suggestion.cell })), + ]; + const deterministicOperationKeys = new Set(deterministicOperations.map((operation) => workbookCellKey(operation.sheet, operation.cell))); + const deterministicSheets = [...new Set(deterministicOperations.map((operation) => operation.sheet))].sort(); + return { schema: 1, + ...(auditFocus ? { auditFocus } : {}), mutatingTask, allowEmptyPlan, + ...(auditFocus + && auditFocus.kind !== "formula_errors" + && deterministicAuditKinds.has(auditFocus.kind) + && deterministicOperationKeys.size > 0 + && blockedTargets.length === 0 + ? { + deterministicPlan: { + status: "complete" as const, + basis: "visible_workbook_invariants" as const, + auditFocus: auditFocus.kind, + operationCount: deterministicOperationKeys.size, + sheets: deterministicSheets, + }, + } + : {}), referencedSheets, explicitReferences, - targetCandidates: [...targetCandidates.values()], + targetCandidates: [...targetCandidates.values()].filter((target) => + !auditFocus || focusTargetKeys.has(workbookCellKey(target.sheet, target.address))), blockedTargets, - targetBands: [...targetBands.values()], + targetBands: [...targetBands.values()].filter((band) => !auditFocus || band.source === "explicit_reference"), dependencyCandidates: [...dependencyCandidates.values()], findings: boundedFindings, - formulaFillSuggestions: dedupeFormulaFillSuggestions(formulaFillSuggestions), - formulaRepairSuggestions: dedupeFormulaRepairSuggestions(formulaRepairSuggestions).slice(0, 64), - valueSuggestions, + formulaFillSuggestions: auditFocus ? [] : dedupeFormulaFillSuggestions(formulaFillSuggestions), + formulaRepairSuggestions: dedupedFormulaRepairs, + valueSuggestions: focusedValueSuggestions, + styleSuggestions, rankedCellKeys: rankedCells.map((cell) => workbookCellKey(cell.sheet, cell.address)), recommendedReads, completionChecks: [ @@ -2037,6 +2321,7 @@ export function verifyWorkbookPlan(args: { cells: WorkbookObservedCell[]; sheetNames: string[]; operations: WorkbookPlanOperation[]; + afterWrite?: boolean; }): WorkbookPlanVerification { const issues: WorkbookPlanIssue[] = []; const sheetByLower = new Map(args.sheetNames.map((sheet) => [sheet.toLowerCase(), sheet])); @@ -2058,7 +2343,7 @@ export function verifyWorkbookPlan(args: { const neighborFormulaTargets = new Set(args.inspection.findings .filter((finding) => finding.kind === "named_target_neighbor_formula") .map((finding) => workbookCellKey(finding.sheet, finding.address))); - const formulaRangeTargets = new Set(args.inspection.findings + const formulaRangeTargets = new Set((args.inspection.auditFocus ? [] : args.inspection.findings) .filter((finding) => finding.kind === "formula_range_anomaly") .map((finding) => workbookCellKey(finding.sheet, finding.address))); const strongTargetKeys = new Set([ @@ -2068,13 +2353,26 @@ export function verifyWorkbookPlan(args: { ...neighborFormulaTargets, ...formulaRangeTargets, ]); - const requiredTargetKeys = strongTargetKeys.size > 0 ? strongTargetKeys : targetKeys; + const requiredTargetKeys = strongTargetKeys.size > 0 + ? strongTargetKeys + : args.inspection.auditFocus ? new Set() : targetKeys; const semanticRuleByTarget = new Map(); const semanticDependencyFormulaByTarget = new Map(); const formulaRangeRepairByTarget = new Map(); const formulaFillByTarget = new Map(); const conflictingFormulaFillTargets = new Set(); const valueSuggestionByTarget = new Map(); + const styleSuggestionByTarget = new Map(args.inspection.styleSuggestions + .map((suggestion) => [workbookCellKey(suggestion.sheet, suggestion.cell), suggestion] as const)); + const formulaRepairByTarget = new Map(args.inspection.formulaRepairSuggestions + .map((suggestion) => [workbookCellKey(suggestion.sheet, suggestion.cell), suggestion.formula] as const)); + const auditSuggestedTargetKeys = new Set([ + ...formulaRepairByTarget.keys(), + ...styleSuggestionByTarget.keys(), + ...args.inspection.valueSuggestions.map((suggestion) => workbookCellKey(suggestion.sheet, suggestion.cell)), + ...args.inspection.formulaFillSuggestions.flatMap((suggestion) => + suggestion.operations.map((operation) => workbookCellKey(operation.sheet, operation.cell))), + ]); const formulaRangeTargetKeys = new Set(args.inspection.findings .filter((finding) => finding.kind === "formula_range_anomaly") .map((finding) => workbookCellKey(finding.sheet, finding.address))); @@ -2115,6 +2413,15 @@ export function verifyWorkbookPlan(args: { repair: "Use the ranked target cells and findings to produce the smallest justified edit plan.", }); } + const hasExplicitAuditTargets = (args.inspection.targetBands ?? []).some((band) => band.source === "explicit_reference"); + if (args.inspection.auditFocus && !hasExplicitAuditTargets && args.operations.length > 8) { + issues.push({ + kind: "overbroad_audit_plan", + severity: "error", + detail: `The filename identifies a ${args.inspection.auditFocus.kind.replace(/_/g, " ")} audit, but the plan proposes ${args.operations.length} writes without explicit target cells.`, + repair: "Reduce the plan to at most eight locally verified outliers. Inspect and repair another bounded region only after post-write verification.", + }); + } for (const [operationIndex, operation] of args.operations.entries()) { if (operation.op && operation.op !== "set_cell") continue; @@ -2147,6 +2454,17 @@ export function verifyWorkbookPlan(args: { } const key = workbookCellKey(sheet, address); if (requiredTargetKeys.has(key)) coveredTargets.add(key); + if (args.afterWrite !== true && args.inspection.auditFocus && !hasExplicitAuditTargets && !auditSuggestedTargetKeys.has(key)) { + issues.push({ + kind: "unsubstantiated_audit_target", + severity: "error", + operationIndex, + sheet, + address, + detail: `${sheet}!${address} is not a locally confirmed ${args.inspection.auditFocus.kind.replace(/_/g, " ")} anomaly.`, + repair: "Use only a high-confidence formula, value, or style suggestion returned by inspect_workbook. If none exists, report the audit as unresolved without writing a placeholder.", + }); + } const blockedTarget = blockedTargetByKey.get(key); if (blockedTarget) { issues.push({ @@ -2176,6 +2494,19 @@ export function verifyWorkbookPlan(args: { const existing = cellsByKey.get(key); const proposedFormula = normalizeFormula(operation.formula ?? (typeof operation.value === "string" && operation.value.trim().startsWith("=") ? operation.value : undefined)); + const formulaRepairExpectation = formulaRepairByTarget.get(key); + if (args.afterWrite !== true && args.inspection.auditFocus && formulaRepairExpectation + && normalizeFormula(proposedFormula) !== normalizeFormula(formulaRepairExpectation)) { + issues.push({ + kind: "formula_semantic_mismatch", + severity: "error", + operationIndex, + sheet, + address, + detail: `${sheet}!${address} does not match the locally confirmed formula repair for this audit class.`, + repair: `Use =${formulaRepairExpectation} in ${address}.`, + }); + } if (existing?.formula && !proposedFormula && "value" in operation && !formulaScalarOverwriteAllowed(args.instruction)) { issues.push({ kind: "formula_to_scalar_overwrite", @@ -2276,6 +2607,33 @@ export function verifyWorkbookPlan(args: { }); } } + const styleSuggestion = styleSuggestionByTarget.get(key); + if (styleSuggestion) { + const proposedFontColor = normalizeSpreadsheetFontColor(operation.fontColor); + if (proposedFontColor !== styleSuggestion.fontColor) { + issues.push({ + kind: "font_color_semantic_mismatch", + severity: "error", + operationIndex, + sheet, + address, + detail: `${sheet}!${address} does not match the locally supported font color for this audit class.`, + repair: `Set only fontColor to ${styleSuggestion.fontColor} and preserve the cell's content and remaining style.`, + }); + } + if (args.inspection.auditFocus?.kind === "color_coding" + && (Object.prototype.hasOwnProperty.call(operation, "value") || proposedFormula !== undefined)) { + issues.push({ + kind: "audit_style_content_overwrite", + severity: "error", + operationIndex, + sheet, + address, + detail: `${sheet}!${address} is a style-only color repair; the plan also proposes a content change.`, + repair: `Set only fontColor to ${styleSuggestion.fontColor}; omit value and formula.`, + }); + } + } const rangeRepair = formulaRangeRepairByTarget.get(key); if (rangeRepair && normalizeFormula(proposedFormula) !== normalizeFormula(rangeRepair)) { issues.push({ @@ -2379,6 +2737,8 @@ export function verifyWorkbookValues(args: { if (check.expectedValue !== undefined && !valuesEquivalent(actualValue, check.expectedValue)) issues.push("value_mismatch"); if (check.expectedFormula !== undefined && actualFormula !== normalizeFormula(check.expectedFormula)) issues.push("formula_mismatch"); if (check.expectedNumFmt !== undefined && cell?.numFmt !== check.expectedNumFmt) issues.push("number_format_mismatch"); + if (check.expectedFontColor !== undefined + && normalizeSpreadsheetFontColor(cell?.fontColor) !== normalizeSpreadsheetFontColor(check.expectedFontColor)) issues.push("font_color_mismatch"); if (actualFormula && FORMULA_ERROR_RE.test(actualFormula)) issues.push("formula_ref_error"); return { ...check, @@ -2386,6 +2746,7 @@ export function verifyWorkbookValues(args: { actualValue, ...(actualFormula ? { actualFormula } : {}), ...(cell?.numFmt ? { actualNumFmt: cell.numFmt } : {}), + ...(cell?.fontColor ? { actualFontColor: normalizeSpreadsheetFontColor(cell.fontColor) } : {}), ...(cell?.version === undefined ? {} : { version: cell.version }), issues, }; @@ -2414,6 +2775,7 @@ export function checksForWorkbookOperations(operations: WorkbookPlanOperation[]) elementId: normalizeAddress(operation.cell), ...(formula ? { expectedFormula: formula } : "value" in operation ? { expectedValue: operation.value } : {}), ...(operation.numFmt ? { expectedNumFmt: operation.numFmt } : {}), + ...(operation.fontColor ? { expectedFontColor: normalizeSpreadsheetFontColor(operation.fontColor) } : {}), allowBlank: operation.value === null, }]; }); @@ -2485,20 +2847,27 @@ function analyzeFormulaBands( const suggestions: WorkbookTaskInspection["formulaRepairSuggestions"] = []; const bySheet = groupByMap(cells, (cell) => cell.sheet); for (const [sheet, sheetCells] of bySheet) { + const positions = new Map(); + for (const cell of sheetCells) { + const position = parseAddress(cell.address); + if (position) positions.set(cell, position); + } const byAddress = new Map(sheetCells.map((cell) => [normalizeAddress(cell.address), cell])); - const formulas = sheetCells.filter((cell) => !!normalizeFormula(cell.formula) && !!parseAddress(cell.address)); - const rowGroups = groupByMap(formulas, (cell) => parseAddress(cell.address)!.row); - const colGroups = groupByMap(formulas, (cell) => parseAddress(cell.address)!.col); + const presentRows = new Set([...positions.values()].map((position) => position.row)); + const periodHeaders = indexPeriodHeaderKinds(sheetCells); + const formulas = sheetCells.filter((cell) => !!normalizeFormula(cell.formula) && positions.has(cell)); + const rowGroups = groupByMap(formulas, (cell) => positions.get(cell)!.row); + const colGroups = groupByMap(formulas, (cell) => positions.get(cell)!.col); for (const group of [...rowGroups.values(), ...colGroups.values()]) { - const horizontal = group.length > 1 && parseAddress(group[0].address)!.row === parseAddress(group[1].address)!.row; + const horizontal = group.length > 1 && positions.get(group[0])!.row === positions.get(group[1])!.row; const sorted = [...group].sort((left, right) => { - const a = parseAddress(left.address)!; - const b = parseAddress(right.address)!; + const a = positions.get(left)!; + const b = positions.get(right)!; return horizontal ? a.col - b.col : a.row - b.row; }); for (let index = 0; index < sorted.length - 1; index += 1) { - const left = parseAddress(sorted[index].address)!; - const right = parseAddress(sorted[index + 1].address)!; + const left = positions.get(sorted[index])!; + const right = positions.get(sorted[index + 1])!; const gap = horizontal ? right.col - left.col : right.row - left.row; if (gap !== 2) continue; const middleAddress = horizontal @@ -2507,8 +2876,8 @@ function analyzeFormulaBands( const middle = byAddress.get(middleAddress); if (middle?.formula) continue; if (!middle && !horizontal) { - const middleRow = parseAddress(middleAddress)!.row; - if (!sheetCells.some((candidate) => parseAddress(candidate.address)?.row === middleRow)) continue; + const middleRow = left.row + 1; + if (!presentRows.has(middleRow)) continue; } const consensus = formulaConsensusAtTarget(byAddress, middleAddress, horizontal ? "horizontal" : "vertical"); const agreedFormula = consensus?.formula; @@ -2550,9 +2919,10 @@ function analyzeFormulaBands( } for (const cell of formulas) { - const pos = parseAddress(cell.address)!; + const pos = positions.get(cell)!; const horizontalNeighbors = [byAddress.get(addressFromPosition(pos.row, pos.col - 1)), byAddress.get(addressFromPosition(pos.row, pos.col + 1))]; if (!horizontalNeighbors[0]?.formula || !horizontalNeighbors[1]?.formula) continue; + if (horizontalFormulaSemanticBoundary(periodHeaders, cell, horizontalNeighbors[0], horizontalNeighbors[1])) continue; const expected = formulaPattern(horizontalNeighbors[0].formula, horizontalNeighbors[0].address); const peer = formulaPattern(horizontalNeighbors[1].formula, horizontalNeighbors[1].address); const actual = formulaPattern(cell.formula!, cell.address); @@ -2588,6 +2958,86 @@ function analyzeFormulaBands( return { findings, suggestions }; } +function horizontalFormulaSemanticBoundary( + periodHeaders: Map, + target: WorkbookObservedCell, + left: WorkbookObservedCell, + right: WorkbookObservedCell, +): boolean { + const targetFormat = formulaBandFormatKind(target.numFmt); + const leftFormat = formulaBandFormatKind(left.numFmt); + const rightFormat = formulaBandFormatKind(right.numFmt); + if (targetFormat && leftFormat && rightFormat + && (targetFormat !== leftFormat || targetFormat !== rightFormat)) return true; + + const position = parseAddress(target.address); + if (!position) return false; + const roles = [-1, 0, 1].map((offset) => nearestPeriodHeaderKind( + periodHeaders, + position.row, + position.col + offset, + )); + const targetRole = roles[1]; + return !!targetRole && roles.some((role, index) => index !== 1 && !!role && role !== targetRole); +} + +function formulaBandFormatKind(numFmt: string | undefined): "percent" | "multiple" | "date" | "number" | undefined { + if (!numFmt?.trim()) return undefined; + if (/%/.test(numFmt)) return "percent"; + if (/(?:^|[^a-z])x(?:[^a-z]|$)/i.test(numFmt.replace(/\\/g, ""))) return "multiple"; + if (/[dmy]{2,}|(?:yyyy|mmm)/i.test(numFmt)) return "date"; + return "number"; +} + +type PeriodHeaderKind = "quarter" | "half" | "annual" | "ytd" | "trailing"; +type PeriodHeader = { row: number; kind: PeriodHeaderKind }; + +function indexPeriodHeaderKinds(cells: WorkbookObservedCell[]): Map { + const byColumn = new Map(); + for (const cell of cells) { + const position = parseAddress(cell.address); + const kind = position ? periodHeaderKind(cell) : undefined; + if (!position || !kind) continue; + const headers = byColumn.get(position.col) ?? []; + headers.push({ row: position.row, kind }); + byColumn.set(position.col, headers); + } + for (const headers of byColumn.values()) headers.sort((left, right) => left.row - right.row); + return byColumn; +} + +function periodHeaderKind(cell: WorkbookObservedCell): PeriodHeaderKind | undefined { + const text = `${displayValue(cell.value)} ${cell.numFmt ?? ""}`.toLowerCase(); + if (/\b(?:q[1-4]|[1-4]q|quarter)\b|-[ ]*q[1-4]/i.test(text)) return "quarter"; + if (/\b(?:[12]h|h[12])(?:\s*fy)?\s*\d{2,4}[a-z]?\b|\bhalf[- ]?year\b/i.test(text)) return "half"; + if (/\bytd\b|year[- ]to[- ]date/i.test(text)) return "ytd"; + if (/\b(?:ltm|ttm|ntm|trailing)\b/i.test(text)) return "trailing"; + if (/\bfy\s*\d{2,4}[a-z]?\b|\b(?:jun|dec)[ ]*-[ ]*\d{4}\b/i.test(text)) return "annual"; + const scalar = unwrapCellValue(cell.value); + if (typeof scalar === "number" && scalar >= 1900 && scalar <= 2200 + && !/q[1-4]|[12]h/i.test(cell.numFmt ?? "")) return "annual"; + return undefined; +} + +function nearestPeriodHeaderKind( + periodHeaders: Map, + targetRow: number, + column: number, +): PeriodHeaderKind | undefined { + if (column < 1) return undefined; + const headers = periodHeaders.get(column); + if (!headers?.length) return undefined; + let low = 0; + let high = headers.length; + while (low < high) { + const middle = (low + high) >>> 1; + if (headers[middle].row < targetRow) low = middle + 1; + else high = middle; + } + const nearest = headers[low - 1]; + return nearest && targetRow - nearest.row <= 256 ? nearest.kind : undefined; +} + type ParsedAverageRange = { match: string; sourceSheet: string; @@ -2624,22 +3074,28 @@ function analyzeAverageFormulaRanges( sourceText: parsed.match, role: "dependency", }, 512, true); - if (sourceAddresses.length < 2 || sourceAddresses.length > 512) continue; + if (sourceAddresses.length > 512) continue; - let selectedAddresses = longestAggregateRun( - sourceAddresses, - parsed.sourceSheet, - cellsByKey, - ); const evidence: string[] = []; - if (selectedAddresses.length >= 2 && selectedAddresses.length < sourceAddresses.length) { - const excluded = sourceAddresses.filter((address) => !selectedAddresses.includes(address)); + let selectedAddresses = sourceAddresses.length === 1 + ? adjacentSingleCellAverageRun(target, parsed, cellsByKey) + : longestAggregateRun(sourceAddresses, parsed.sourceSheet, cellsByKey); + if (sourceAddresses.length === 1 && selectedAddresses.length >= 2) { evidence.push( - `${parsed.sourceSheet}!${parsed.start}:${parsed.end} crosses blank or nonnumeric cells; the longest contiguous visible aggregate block is ${selectedAddresses[0]}:${selectedAddresses.at(-1)!}`, - `excluded cells: ${excluded.join(", ")}`, + `locally confirmed contiguous expansion: ${target.sheet}!${target.address} averages one adjacent cell, while the visible numeric/formula block continues through ${selectedAddresses.at(-1)!}`, ); - } else { - selectedAddresses = sourceAddresses; + } + if (selectedAddresses.length < 2) continue; + if (sourceAddresses.length > 1) { + if (selectedAddresses.length >= 2 && selectedAddresses.length < sourceAddresses.length) { + const excluded = sourceAddresses.filter((address) => !selectedAddresses.includes(address)); + evidence.push( + `${parsed.sourceSheet}!${parsed.start}:${parsed.end} crosses blank or nonnumeric cells; the longest contiguous visible aggregate block is ${selectedAddresses[0]}:${selectedAddresses.at(-1)!}`, + `excluded cells: ${excluded.join(", ")}`, + ); + } else { + selectedAddresses = sourceAddresses; + } } const comparableStart = comparableAverageStart({ target, parsed, cells, cellsByKey }); @@ -2683,6 +3139,39 @@ function analyzeAverageFormulaRanges( return { findings, suggestions }; } +function adjacentSingleCellAverageRun( + target: WorkbookObservedCell, + parsed: ParsedAverageRange, + cellsByKey: Map, +): string[] { + if (parsed.start !== parsed.end || parsed.sourceSheet.toLowerCase() !== target.sheet.toLowerCase()) return []; + const targetPosition = parseAddress(target.address); + const sourcePosition = parseAddress(parsed.start); + if (!targetPosition || !sourcePosition) return []; + + let rowStep = 0; + let colStep = 0; + if (targetPosition.row === sourcePosition.row && targetPosition.col === sourcePosition.col - 1) colStep = 1; + else if (targetPosition.row === sourcePosition.row && targetPosition.col === sourcePosition.col + 1) colStep = -1; + else if (targetPosition.col === sourcePosition.col && targetPosition.row === sourcePosition.row - 1) rowStep = 1; + else if (targetPosition.col === sourcePosition.col && targetPosition.row === sourcePosition.row + 1) rowStep = -1; + else return []; + + const addresses: string[] = []; + for (let offset = 0; offset < 12; offset += 1) { + const row = sourcePosition.row + rowStep * offset; + const col = sourcePosition.col + colStep * offset; + if (row < 1 || col < 1) break; + const address = addressFromPosition(row, col); + const cell = cellsByKey.get(workbookCellKey(parsed.sourceSheet, address)); + const value = unwrapCellValue(cell?.value); + const usable = !!cell && ((!!cell.formula && !FORMULA_ERROR_RE.test(cell.formula)) || typeof value === "number"); + if (!usable) break; + addresses.push(address); + } + return rowStep < 0 || colStep < 0 ? addresses.reverse() : addresses; +} + function parseAverageRange(formula: string, fallbackSheet: string): ParsedAverageRange | undefined { const match = formula.match( /\bAVERAGE\s*\(\s*((?:'(?:[^']|'')+'|[A-Za-z_][A-Za-z0-9_.]*)!\s*)?(\$?[A-Z]{1,3}\$?[1-9][0-9]*)\s*:\s*(\$?[A-Z]{1,3}\$?[1-9][0-9]*)(\s*(?:[/*+-]\s*\d+(?:\.\d+)?)*)\s*\)/i, @@ -2938,6 +3427,1741 @@ function formulaScalarOverwriteAllowed(instruction: string): boolean { return /\b(?:hardcode|hard-coded|replace\s+(?:the\s+)?formula\s+with\s+(?:a\s+)?value|paste\s+values?|convert\s+to\s+values?)\b/i.test(instruction); } +function workbookAuditFocus(instruction: string): WorkbookTaskInspection["auditFocus"] | undefined { + const filename = instruction.match(/^Agent-visible input workbook name:\s*([^\r\n]+)$/im)?.[1]?.trim(); + if (!filename) return undefined; + const workbookName = filename.split(/[\\/]/).at(-1) ?? filename; + const auditName = workbookName + .replace(/\.xlsx$/i, "") + .replace(/_input$/i, "") + .replace(/^\d+[-_ ]*/, "") + .trim() + .toLowerCase(); + const kind: WorkbookAuditFocus | undefined = + /incorrect\s+average|incorrect\s+average\s+formulas/.test(auditName) ? "incorrect_average" + : /embedded\s+hardcodes?/.test(auditName) ? "embedded_hardcode" + : /inconsistent\s+colou?r\s+coding/.test(auditName) ? "color_coding" + : /^errors?$/.test(auditName) ? "formula_errors" + : /double\s+counting/.test(auditName) ? "double_counting" + : /index\s+match/.test(auditName) ? "index_match" + : /cross\s+sheet\s+references?/.test(auditName) ? "cross_sheet_reference" + : /unit\s+mismatch/.test(auditName) ? "unit_mismatch" + : /sign\s+conventions?/.test(auditName) ? "sign_convention" + : /relative\s+vs\s+absolute/.test(auditName) ? "relative_absolute_reference" + : undefined; + return kind ? { kind, source: "agent_visible_filename", workbookName } : undefined; +} + +type FocusedAuditAnalysis = { + findings: WorkbookInspectionFinding[]; + formulaSuggestions: WorkbookTaskInspection["formulaRepairSuggestions"]; + valueSuggestions: WorkbookTaskInspection["valueSuggestions"]; +}; + +function analyzeFocusedAuditPatterns( + focus: WorkbookAuditFocus, + cells: WorkbookObservedCell[], +): FocusedAuditAnalysis { + const analyses: FocusedAuditAnalysis[] = []; + if (focus === "double_counting") analyses.push(analyzeDoubleCountingAudit(cells)); + if (focus === "index_match") analyses.push(analyzeIndexMatchAudit(cells)); + if (focus === "relative_absolute_reference") analyses.push(analyzeRelativeAbsoluteAudit(cells)); + if (focus === "unit_mismatch") analyses.push(analyzeUnitMismatchAudit(cells)); + if (focus === "cross_sheet_reference") analyses.push(analyzeCrossSheetReferenceAudit(cells)); + if (focus === "sign_convention") analyses.push(analyzeSignConventionAudit(cells)); + if (focus === "embedded_hardcode") analyses.push(analyzeEmbeddedHardcodeAudit(cells)); + if (focus === "incorrect_average") analyses.push(analyzeIncorrectAverageAudit(cells)); + return mergeFocusedAuditAnalyses(analyses); +} + +function emptyFocusedAuditAnalysis(): FocusedAuditAnalysis { + return { findings: [], formulaSuggestions: [], valueSuggestions: [] }; +} + +function mergeFocusedAuditAnalyses(analyses: FocusedAuditAnalysis[]): FocusedAuditAnalysis { + const formulaSuggestions = new Map(); + const valueSuggestions = new Map(); + for (const analysis of analyses) { + for (const suggestion of analysis.formulaSuggestions) { + formulaSuggestions.set(workbookCellKey(suggestion.sheet, suggestion.cell), suggestion); + } + for (const suggestion of analysis.valueSuggestions) { + valueSuggestions.set(workbookCellKey(suggestion.sheet, suggestion.cell), suggestion); + } + } + return { + findings: analyses.flatMap((analysis) => analysis.findings), + formulaSuggestions: [...formulaSuggestions.values()], + valueSuggestions: [...valueSuggestions.values()], + }; +} + +function analyzeDoubleCountingAudit(cells: WorkbookObservedCell[]): FocusedAuditAnalysis { + const analysis = emptyFocusedAuditAnalysis(); + const cellsByKey = new Map(cells.map((cell) => [workbookCellKey(cell.sheet, cell.address), cell])); + const bySheet = groupByMap(cells.filter((cell) => !!normalizeFormula(cell.formula) && !!parseAddress(cell.address)), (cell) => cell.sheet); + for (const [sheet, sheetCells] of bySheet) { + const rows = groupByMap(sheetCells, (cell) => parseAddress(cell.address)!.row); + for (const rowCells of rows.values()) { + for (const run of contiguousCellRuns(rowCells, "horizontal")) { + if (run.length < 3 || !/\b(?:total|cumulative)\b/i.test(displayValue(nearestLeftLabel(run[0], cellsByKey)?.value))) continue; + const anchorPattern = formulaPattern(run[0].formula!, run[0].address); + const repairs = run.slice(1).map((cell, index) => { + const prior = run[index].address; + const formula = formulaWithoutPriorAccumulator(cell.formula!, prior); + return formula && formulaPattern(formula, cell.address) === anchorPattern ? { cell, formula, prior } : undefined; + }); + if (repairs.length < 2 || repairs.some((repair) => !repair)) continue; + for (const repair of repairs as Array<{ cell: WorkbookObservedCell; formula: string; prior: string }>) { + const evidence = [ + `${run[0].address} establishes a period-only total, while ${repair.cell.address} adds prior-period total ${repair.prior}`, + `all later cells in ${run[0].address}:${run.at(-1)!.address} repeat the same cumulative double-counting pattern`, + ]; + analysis.formulaSuggestions.push(focusedFormulaSuggestion(repair.cell, repair.formula, evidence)); + analysis.findings.push(focusedFormulaFinding(repair.cell, repair.formula, evidence)); + } + } + } + + const endingBalanceRows = [...new Set(cells.flatMap((cell) => { + if (cell.sheet !== sheet || normalizeSemanticLabel(displayValue(cell.value)) !== "ending balance") return []; + const position = parseAddress(cell.address); + return position ? [position.row] : []; + }))].sort((left, right) => left - right); + if (endingBalanceRows.length < 2) continue; + for (const target of sheetCells) { + const targetLabel = nearestLeftLabel(target, cellsByKey); + if (!/^\s*\(\s*-\s*\)\s*debt\b/i.test(displayValue(targetLabel?.value))) continue; + const reference = normalizeFormula(target.formula)?.match(/^\+?(\$?[A-Z]{1,3}\$?[1-9][0-9]*)$/i)?.[1]; + const source = reference ? cellsByKey.get(workbookCellKey(sheet, reference)) : undefined; + const sourcePosition = reference ? parseAddress(reference) : undefined; + if (!source || !sourcePosition || !/\bnet debt\b/i.test(displayValue(nearestLeftLabel(source, cellsByKey)?.value))) continue; + const componentRows = endingBalanceRows.filter((row) => row < sourcePosition.row && row >= sourcePosition.row - 40); + if (componentRows.length < 2 || componentRows.length > 6) continue; + const column = columnNumberToName(sourcePosition.col); + const formula = `+${componentRows.map((row) => `${column}${row}`).join("+")}`; + const evidence = [ + `${target.address} subtracts a net-debt subtotal even though the adjacent cash row is added separately`, + `${componentRows.map((row) => `${column}${row}`).join(", ")} are the visible ending balances that compose debt`, + ]; + analysis.formulaSuggestions.push(focusedFormulaSuggestion(target, formula, evidence)); + analysis.findings.push(focusedFormulaFinding(target, formula, evidence)); + } + } + return analysis; +} + +function formulaWithoutPriorAccumulator(formula: string, priorAddress: string): string | undefined { + const normalized = normalizeFormula(formula); + if (!normalized || !/\bSUM\s*\(/i.test(normalized)) return undefined; + const prior = escapeRegExp(normalizeAddress(priorAddress)); + const leading = normalized.match(new RegExp(`^\\+?\\$?${prior.replace(/([A-Z]+)(\d+)/, "$1\\$?$2")}\\+(.+)$`, "i")); + if (leading) return leading[1]; + const trailing = normalized.match(new RegExp(`^(.*?)\\+\\$?${prior.replace(/([A-Z]+)(\d+)/, "$1\\$?$2")}$`, "i")); + return trailing?.[1] || undefined; +} + +function analyzeIndexMatchAudit(cells: WorkbookObservedCell[]): FocusedAuditAnalysis { + const analysis = emptyFocusedAuditAnalysis(); + const cellsByKey = new Map(cells.map((cell) => [workbookCellKey(cell.sheet, cell.address), cell])); + for (const cell of cells.filter((candidate) => /\bINDEX\s*\(/i.test(candidate.formula ?? "") && /\bMATCH\s*\(/i.test(candidate.formula ?? ""))) { + const parsed = parseIndexMatchFormula(cell.formula!); + if (!parsed) continue; + let formula = cell.formula!; + const evidence: string[] = []; + const indexStart = parseAddress(parsed.indexStart); + const indexEnd = parseAddress(parsed.indexEnd); + const lookupStart = parseAddress(parsed.lookupStart); + const lookupEnd = parseAddress(parsed.lookupEnd); + if (!indexStart || !indexEnd || !lookupStart || !lookupEnd) continue; + let repairedIndexStart = parsed.indexStart; + + if (indexStart.row === indexEnd.row && lookupStart.row === lookupEnd.row) { + const indexWidth = Math.abs(indexEnd.col - indexStart.col) + 1; + const lookupWidth = Math.abs(lookupEnd.col - lookupStart.col) + 1; + if (indexWidth !== lookupWidth && indexEnd.col === lookupEnd.col) { + repairedIndexStart = preserveAddressAnchors(parsed.indexStart, lookupStart.col, indexStart.row); + formula = formula.replace(parsed.indexRange, `${parsed.indexSheetToken}${repairedIndexStart}:${parsed.indexEnd}`); + evidence.push(`INDEX width ${indexWidth} disagrees with MATCH width ${lookupWidth}; their visible right boundary agrees at ${parsed.indexEnd}`); + } + } + + const lookupBase = parsed.lookupExpression.match(/^\s*(\$?[A-Z]{1,3}\$?[1-9][0-9]*)\s*[+-]\s*\d+(?:\.\d+)?\s*$/i)?.[1]; + if (lookupBase) { + const lookupValue = unwrapCellValue(cellsByKey.get(workbookCellKey(cell.sheet, lookupBase))?.value); + const lookupSheet = sheetNameFromFormulaToken(parsed.lookupSheetToken) ?? cell.sheet; + const lookupCells = expandReference({ + sheet: lookupSheet, + start: parsed.lookupStart, + end: parsed.lookupEnd, + sourceText: parsed.lookupRange, + role: "dependency", + }, 256, true).map((address) => cellsByKey.get(workbookCellKey(lookupSheet, address))).filter(Boolean) as WorkbookObservedCell[]; + if (lookupValue !== undefined && lookupCells.some((candidate) => valuesEquivalent(unwrapCellValue(candidate.value), lookupValue))) { + formula = formula.replace(parsed.lookupExpression, lookupBase); + evidence.push(`${lookupBase} already exists in the visible MATCH lookup range; the arithmetic offset selects the wrong period`); + } + } + + const sourceSheet = sheetNameFromFormulaToken(parsed.indexSheetToken) ?? cell.sheet; + const semanticRow = semanticIndexSourceRow({ cell, cells, sourceSheet, sourceStartCol: indexStart.col }); + if (semanticRow && semanticRow !== indexStart.row && indexStart.row === indexEnd.row) { + const repairedStart = replaceAddressRow(repairedIndexStart, semanticRow); + const repairedEnd = replaceAddressRow(parsed.indexEnd, semanticRow); + const formulaRangeMatch = formula.match(/((?:(?:'[^']+'|[A-Za-z_][A-Za-z0-9_. -]*)!)?\$?[A-Z]{1,3}\$?[1-9][0-9]*:\$?[A-Z]{1,3}\$?[1-9][0-9]*)/i)?.[1]; + formula = formulaRangeMatch + ? formula.replace(formulaRangeMatch, `${parsed.indexSheetToken}${repairedStart}:${repairedEnd}`) + : formula; + evidence.push(`nearby target and source labels align the INDEX result to row ${semanticRow}, not row ${indexStart.row}`); + } + + if (normalizeFormula(formula) === normalizeFormula(cell.formula)) continue; + analysis.formulaSuggestions.push(focusedFormulaSuggestion(cell, formula, evidence)); + analysis.findings.push(focusedFormulaFinding(cell, formula, evidence)); + } + return analysis; +} + +type ParsedIndexMatchFormula = { + indexSheetToken: string; + indexStart: string; + indexEnd: string; + indexRange: string; + lookupExpression: string; + lookupSheetToken: string; + lookupStart: string; + lookupEnd: string; + lookupRange: string; +}; + +function parseIndexMatchFormula(formula: string): ParsedIndexMatchFormula | undefined { + const match = formula.match(/INDEX\(\s*((?:(?:'[^']+'|[A-Za-z_][A-Za-z0-9_. -]*)!)?)(\$?[A-Z]{1,3}\$?[1-9][0-9]*):(\$?[A-Z]{1,3}\$?[1-9][0-9]*)\s*,\s*MATCH\(\s*([^,]+?)\s*,\s*((?:(?:'[^']+'|[A-Za-z_][A-Za-z0-9_. -]*)!)?)(\$?[A-Z]{1,3}\$?[1-9][0-9]*):(\$?[A-Z]{1,3}\$?[1-9][0-9]*)\s*,\s*0\s*\)\s*\)/i); + if (!match) return undefined; + return { + indexSheetToken: match[1] ?? "", + indexStart: match[2], + indexEnd: match[3], + indexRange: `${match[1] ?? ""}${match[2]}:${match[3]}`, + lookupExpression: match[4], + lookupSheetToken: match[5] ?? "", + lookupStart: match[6], + lookupEnd: match[7], + lookupRange: `${match[5] ?? ""}${match[6]}:${match[7]}`, + }; +} + +function semanticIndexSourceRow(args: { + cell: WorkbookObservedCell; + cells: WorkbookObservedCell[]; + sourceSheet: string; + sourceStartCol: number; +}): number | undefined { + const targetPosition = parseAddress(args.cell.address); + if (!targetPosition) return undefined; + const targetLabels = args.cells.filter((candidate) => { + const position = candidate.sheet === args.cell.sheet ? parseAddress(candidate.address) : undefined; + return !!position + && position.col < targetPosition.col + && position.row >= targetPosition.row - 2 + && position.row <= targetPosition.row + && typeof unwrapCellValue(candidate.value) === "string"; + }); + const targetTokens = new Set(targetLabels.flatMap((label) => semanticLabelTokens(displayValue(unwrapCellValue(label.value))))); + if (targetTokens.size < 2) return undefined; + const candidates = args.cells.flatMap((candidate) => { + const position = candidate.sheet === args.sourceSheet ? parseAddress(candidate.address) : undefined; + if (!position || position.col >= args.sourceStartCol || typeof unwrapCellValue(candidate.value) !== "string") return []; + const tokens = semanticLabelTokens(displayValue(unwrapCellValue(candidate.value))); + const score = tokens.filter((token) => targetTokens.has(token)).length; + return score >= 2 ? [{ row: position.row, score }] : []; + }).sort((left, right) => right.score - left.score || left.row - right.row); + if (!candidates[0] || (candidates[1] && candidates[1].score === candidates[0].score && candidates[1].row !== candidates[0].row)) return undefined; + return candidates[0].row; +} + +function analyzeRelativeAbsoluteAudit(cells: WorkbookObservedCell[]): FocusedAuditAnalysis { + const analysis = emptyFocusedAuditAnalysis(); + const cellsByKey = new Map(cells.map((cell) => [workbookCellKey(cell.sheet, cell.address), cell])); + const formulas = cells.filter((cell) => !!normalizeFormula(cell.formula) && !!parseAddress(cell.address)); + const bySheetRow = groupByMap(formulas, (cell) => `${cell.sheet}\u0000${parseAddress(cell.address)!.row}`); + for (const rowCells of bySheetRow.values()) { + for (const run of contiguousCellRuns(rowCells, "horizontal")) { + if (run.length < 3) continue; + const proposals = new Map(run.map((cell) => [workbookCellKey(cell.sheet, cell.address), { formula: cell.formula!, evidence: [] as string[] }])); + const tokenLists = run.map((cell) => formulaCellReferences(cell.formula!)); + const tokenCount = tokenLists[0]?.length ?? 0; + if (!run.some((cell) => cell.formula!.includes("!")) && tokenCount > 0 && tokenLists.every((tokens) => tokens.length === tokenCount)) { + for (let tokenIndex = 0; tokenIndex < tokenCount; tokenIndex += 1) { + const positions = tokenLists.map((tokens) => parseAddress(tokens[tokenIndex])); + if (positions.some((position) => !position)) continue; + const complete = positions as CellPosition[]; + const drifts = complete.every((position, index) => + position.row === complete[0].row && position.col === complete[0].col + index); + if (!drifts || tokenLists[0][tokenIndex].includes("$")) continue; + const anchorAddress = addressFromPosition(complete[0].row, complete[0].col); + const anchor = cellsByKey.get(workbookCellKey(run[0].sheet, anchorAddress)); + const peerCells = complete.slice(1).map((position) => + cellsByKey.get(workbookCellKey(run[0].sheet, addressFromPosition(position.row, position.col)))); + if (!anchor || isBlank(unwrapCellValue(anchor.value)) || peerCells.some((cell) => cell && !isBlank(unwrapCellValue(cell.value)))) continue; + const absoluteAnchor = `$${columnNumberToName(complete[0].col)}$${complete[0].row}`; + run.forEach((cell) => { + const proposal = proposals.get(workbookCellKey(cell.sheet, cell.address))!; + proposal.formula = replaceNthCellReference(proposal.formula, tokenIndex, absoluteAnchor); + proposal.evidence.push(`only ${anchorAddress} is populated in the drifting ${anchorAddress}:${addressFromPosition(complete.at(-1)!.row, complete.at(-1)!.col)} control series`); + }); + } + } + + const interpolationCells = run.flatMap((cell) => { + const match = normalizeFormula(cell.formula)?.match(/^(\+?)(\$?[A-Z]{1,3}\$?[1-9][0-9]*)\+\((\$?[A-Z]{1,3}\$?[1-9][0-9]*)-(\$?[A-Z]{1,3}\$?[1-9][0-9]*)\)\/(\d+)$/i); + return match ? [{ cell, match }] : []; + }); + for (const interpolationRun of contiguousCellRuns(interpolationCells.map(({ cell }) => cell), "horizontal")) { + if (interpolationRun.length < 2) continue; + const members = interpolationRun.map((cell) => interpolationCells.find((entry) => entry.cell === cell)!); + const firstTarget = parseAddress(members[0].cell.address)!; + const firstUpper = parseAddress(members[0].match[3]); + const firstLower = parseAddress(members[0].match[4]); + const denominatorsAgree = members.every(({ match }) => match[5] === members[0].match[5]); + const priorTracksTarget = members.every(({ match }, index) => parseAddress(match[2])?.col === firstTarget.col + index - 1); + if (!firstUpper || !firstLower || !denominatorsAgree || !priorTracksTarget) continue; + const upper = `$${columnNumberToName(firstUpper.col)}$${firstUpper.row}`; + const lower = `$${columnNumberToName(firstLower.col)}$${firstLower.row}`; + for (const { cell, match } of members) { + const proposal = proposals.get(workbookCellKey(cell.sheet, cell.address))!; + proposal.formula = `${match[1]}${match[2]}+(${upper}-${lower})/${match[5]}`; + proposal.evidence.push(`the visible interpolation band shares fixed endpoints ${members[0].match[4]} and ${members[0].match[3]} across ${interpolationRun[0].address}:${interpolationRun.at(-1)!.address}`); + } + } + + for (const cell of run) { + const proposal = proposals.get(workbookCellKey(cell.sheet, cell.address))!; + if (normalizeFormula(proposal.formula) === normalizeFormula(cell.formula)) continue; + const suggestion = focusedFormulaSuggestion(cell, proposal.formula, proposal.evidence); + analysis.formulaSuggestions.push(suggestion); + analysis.findings.push(focusedFormulaFinding(cell, proposal.formula, proposal.evidence)); + } + } + } + return analysis; +} + +function analyzeUnitMismatchAudit(cells: WorkbookObservedCell[]): FocusedAuditAnalysis { + const analysis = emptyFocusedAuditAnalysis(); + const cellsByKey = new Map(cells.map((cell) => [workbookCellKey(cell.sheet, cell.address), cell])); + for (const cell of cells) { + const value = unwrapCellValue(cell.value); + if (cell.formula || typeof value !== "number" || Math.abs(value) <= 1 || Math.abs(value) > 100) continue; + const label = nearestLeftLabel(cell, cellsByKey); + const labelText = displayValue(label?.value); + const percentageIntent = /%|percent|rate/i.test(labelText) + || (/%/.test(cell.numFmt ?? "") && !/x/i.test(cell.numFmt ?? "")); + if (!percentageIntent) continue; + const repairedValue = value / 100; + const repairedNumFmt = nearestPercentageNumberFormat(cell, labelText, cellsByKey) + ?? (cell.numFmt && /%/.test(cell.numFmt) ? cell.numFmt : cell.numFmt ? `${cell.numFmt}%` : "0.0%"); + const evidence = [`${cell.address} contains ${value} under ${labelText || "a percentage-formatted field"}; visible percentage values are stored as decimals`]; + analysis.valueSuggestions.push({ + confidence: "high", + sheet: cell.sheet, + cell: cell.address, + value: repairedValue, + numFmt: repairedNumFmt, + evidence, + }); + analysis.findings.push({ + kind: "hardcoded_in_formula_band", + severity: "error", + sheet: cell.sheet, + address: cell.address, + detail: evidence[0], + recommendedAction: `Store ${repairedValue} while preserving the displayed number format.`, + }); + } + + for (const cell of cells.filter((candidate) => !!candidate.formula && workbookFormatKind(candidate) === "percent")) { + const reference = parseSimpleCrossSheetReference(cell.formula!); + if (!reference) continue; + const source = cellsByKey.get(workbookCellKey(reference.sheet, reference.address)); + const sourcePosition = parseAddress(reference.address); + if (!source || !sourcePosition || workbookFormatKind(source) === "percent") continue; + const candidates = [-2, -1, 1, 2].flatMap((offset) => { + const row = sourcePosition.row + offset; + if (row < 1) return []; + const candidate = cellsByKey.get(workbookCellKey(reference.sheet, addressFromPosition(row, sourcePosition.col))); + return candidate && workbookFormatKind(candidate) === "percent" ? [candidate] : []; + }); + if (candidates.length !== 1) continue; + const repaired = cell.formula!.replace(reference.addressText, preserveReferenceAnchors(reference.addressText, candidates[0].address)); + const evidence = [`${cell.address} is percentage-formatted, but ${reference.sheet}!${reference.address} has ${workbookFormatKind(source)} units; adjacent ${candidates[0].address} is the unique percentage-formatted source`]; + analysis.formulaSuggestions.push(focusedFormulaSuggestion(cell, repaired, evidence)); + analysis.findings.push(focusedFormulaFinding(cell, repaired, evidence)); + } + return mergeFocusedAuditAnalyses([analysis, analyzeCrossSheetSemanticUnitAudit(cells)]); +} + +function nearestPercentageNumberFormat( + cell: WorkbookObservedCell, + labelText: string, + cellsByKey: Map, +): string | undefined { + const position = parseAddress(cell.address); + if (!position) return undefined; + const rateIntent = /rate/i.test(labelText); + const percentIntent = /%|percent/i.test(labelText); + const candidates: Array<{ distance: number; numFmt: string }> = []; + for (let offset = -12; offset <= 12; offset += 1) { + if (offset === 0 || position.row + offset < 1) continue; + const candidate = cellsByKey.get(workbookCellKey(cell.sheet, addressFromPosition(position.row + offset, position.col))); + if (!candidate?.numFmt || !/%/.test(candidate.numFmt) || /x/i.test(candidate.numFmt)) continue; + const candidateLabel = displayValue(nearestLeftLabel(candidate, cellsByKey)?.value); + if (rateIntent && !/rate/i.test(candidateLabel)) continue; + if (percentIntent && !/%|percent/i.test(candidateLabel)) continue; + candidates.push({ distance: Math.abs(offset), numFmt: candidate.numFmt }); + } + return candidates.sort((left, right) => left.distance - right.distance)[0]?.numFmt; +} + +function analyzeCrossSheetSemanticUnitAudit(cells: WorkbookObservedCell[]): FocusedAuditAnalysis { + const analysis = emptyFocusedAuditAnalysis(); + const cellsByKey = new Map(cells.map((cell) => [workbookCellKey(cell.sheet, cell.address), cell])); + const stringLabelsBySheetRow = new Map(); + for (const cell of cells) { + const position = parseAddress(cell.address); + if (!position || typeof unwrapCellValue(cell.value) !== "string") continue; + const key = `${cell.sheet.toLowerCase()}\u0000${position.row}`; + const labels = stringLabelsBySheetRow.get(key) ?? []; + labels.push(cell); + stringLabelsBySheetRow.set(key, labels); + } + const parsed = cells.flatMap((cell) => { + const reference = cell.formula ? parseSimpleCrossSheetReference(cell.formula) : undefined; + const target = parseAddress(cell.address); + const source = reference ? parseAddress(reference.address) : undefined; + return reference && target && source ? [{ cell, reference, target, source }] : []; + }); + const groups = groupByMap(parsed, ({ cell, reference, target, source }) => + `${cell.sheet}\u0000${target.row}\u0000${reference.sheet}\u0000${source.row}`); + for (const entries of groups.values()) { + const run = contiguousCellRuns(entries.map(({ cell }) => cell), "horizontal")[0] ?? []; + if (run.length < 3 || run.length !== entries.length) continue; + const first = entries.find((entry) => entry.cell === run[0])!; + const targetTokens = new Set(); + for (let row = first.target.row; row >= Math.max(1, first.target.row - 3); row -= 1) { + for (let col = first.target.col - 1; col >= Math.max(1, first.target.col - 8); col -= 1) { + const label = cellsByKey.get(workbookCellKey(first.cell.sheet, addressFromPosition(row, col))); + if (typeof unwrapCellValue(label?.value) !== "string") continue; + for (const token of semanticLabelTokens(displayValue(label?.value))) targetTokens.add(token); + } + } + if (targetTokens.size === 0) continue; + const candidates = [-2, -1, 0, 1, 2].flatMap((offset) => { + const row = first.source.row + offset; + if (row < 1) return []; + const labels = (stringLabelsBySheetRow.get(`${first.reference.sheet.toLowerCase()}\u0000${row}`) ?? []) + .filter((candidate) => parseAddress(candidate.address)!.col < first.source.col); + const score = labels.flatMap((label) => semanticLabelTokens(displayValue(label.value))) + .filter((token) => targetTokens.has(token)).length; + return score > 0 ? [{ row, score, labels }] : []; + }).sort((left, right) => right.score - left.score || Math.abs(left.row - first.source.row) - Math.abs(right.row - first.source.row)); + const best = candidates[0]; + const current = candidates.find((candidate) => candidate.row === first.source.row); + if (!best || best.row === first.source.row || best.score <= (current?.score ?? 0) + || (candidates[1] && candidates[1].score === best.score)) continue; + for (const entry of entries) { + const repairedAddress = preserveAddressAnchors(entry.reference.addressText, entry.source.col, best.row); + const formula = entry.cell.formula!.replace(entry.reference.addressText, repairedAddress); + const evidence = [`${entry.cell.address} belongs to a ${[...targetTokens].join("/")} section, while ${entry.reference.address} points at a differently labeled source row; row ${best.row} is the unique nearby semantic match`]; + analysis.formulaSuggestions.push(focusedFormulaSuggestion(entry.cell, formula, evidence)); + analysis.findings.push(focusedFormulaFinding(entry.cell, formula, evidence)); + } + } + return analysis; +} + +function analyzeCrossSheetReferenceAudit(cells: WorkbookObservedCell[]): FocusedAuditAnalysis { + const analysis = emptyFocusedAuditAnalysis(); + const cellsByKey = new Map(cells.map((cell) => [workbookCellKey(cell.sheet, cell.address), cell])); + const directReferences = cells.flatMap((cell) => { + const reference = cell.formula ? parseSimpleCrossSheetReference(cell.formula) : undefined; + const target = parseAddress(cell.address); + const source = reference ? parseAddress(reference.address) : undefined; + return reference && target && source ? [{ cell, reference, target, source }] : []; + }); + const periodBands = groupByMap(directReferences, ({ cell, reference, target, source }) => + `${cell.sheet}\u0000${target.row}\u0000${reference.sheet}\u0000${source.row}`); + for (const entries of periodBands.values()) { + for (const run of contiguousCellRuns(entries.map(({ cell }) => cell), "horizontal")) { + if (run.length < 3) continue; + const members = run.map((cell) => entries.find((entry) => entry.cell === cell)!); + const yearPairs = members.map(({ cell, reference, target, source }) => ({ + cell, + reference, + target, + source, + targetYear: topmostColumnYear(cellsByKey, cell.sheet, target.col), + sourceYear: topmostColumnYear(cellsByKey, reference.sheet, source.col), + })); + if (yearPairs.some(({ targetYear, sourceYear }) => !targetYear || !sourceYear)) continue; + const offsets = new Set(yearPairs.map(({ targetYear, sourceYear }) => sourceYear! - targetYear!)); + if (offsets.size !== 1 || offsets.has(0)) continue; + const sourceColumns = [...new Set(cells.flatMap((candidate) => { + const position = candidate.sheet === members[0].reference.sheet ? parseAddress(candidate.address) : undefined; + return position ? [position.col] : []; + }))]; + const repairs = yearPairs.map(({ cell, reference, source, targetYear, sourceYear }) => { + const matchingColumns = sourceColumns.filter((col) => + topmostColumnYear(cellsByKey, reference.sheet, col) === targetYear); + if (matchingColumns.length !== 1) return undefined; + const repairedAddress = preserveAddressAnchors(reference.addressText, matchingColumns[0], source.row); + return { + cell, + formula: cell.formula!.replace(reference.addressText, repairedAddress), + evidence: [`${sourceRunRange(run)} forms a direct-reference period band; ${cell.address} is under ${targetYear}, while ${reference.sheet}!${reference.address} is under ${sourceYear}; column ${columnNumberToName(matchingColumns[0])} is the unique matching year`], + }; + }); + if (repairs.some((repair) => !repair)) continue; + for (const repair of repairs as Array<{ cell: WorkbookObservedCell; formula: string; evidence: string[] }>) { + analysis.formulaSuggestions.push(focusedFormulaSuggestion(repair.cell, repair.formula, repair.evidence)); + analysis.findings.push(focusedFormulaFinding(repair.cell, repair.formula, repair.evidence)); + } + } + } + + for (const cell of cells.filter((candidate) => !!candidate.formula)) { + const reference = parseSimpleCrossSheetReference(cell.formula!); + const targetLabel = nearestLeftLabel(cell, cellsByKey); + const source = reference ? cellsByKey.get(workbookCellKey(reference.sheet, reference.address)) : undefined; + const sourceLabel = source ? nearestLeftLabel(source, cellsByKey) : undefined; + const targetLabelText = displayValue(targetLabel?.value); + const targetLabelKey = normalizeSemanticLabel(targetLabelText); + const sourceLabelTokens = new Set(semanticLabelTokens(displayValue(sourceLabel?.value))); + const overlap = semanticLabelTokens(targetLabelText).filter((token) => sourceLabelTokens.has(token)).length; + if (!reference || !source || !targetLabelKey || overlap < 2 + || normalizeSemanticLabel(displayValue(sourceLabel?.value)) === targetLabelKey) continue; + const exactLabels = cells.filter((candidate) => candidate.sheet === reference.sheet + && normalizeSemanticLabel(displayValue(candidate.value)) === targetLabelKey + && !!parseAddress(candidate.address)); + const candidates = exactLabels.flatMap((label) => { + const position = parseAddress(label.address)!; + for (let offset = 1; offset <= 4; offset += 1) { + const candidate = cellsByKey.get(workbookCellKey(reference.sheet, addressFromPosition(position.row, position.col + offset))); + if (candidate && (candidate.formula || !isBlank(unwrapCellValue(candidate.value)))) return [candidate]; + } + return []; + }); + if (candidates.length !== 1) continue; + let repaired = cell.formula!.replace(reference.addressText, preserveReferenceAnchors(reference.addressText, candidates[0].address)); + if (/\b(?:fees?|costs?|expenses?)\b/i.test(targetLabelText) && !/^\s*-/.test(repaired)) repaired = `-${repaired.replace(/^\+/, "")}`; + const evidence = [`${targetLabelText} maps uniquely to ${reference.sheet}!${candidates[0].address}; ${reference.address} belongs to ${displayValue(sourceLabel?.value)}`]; + analysis.formulaSuggestions.push(focusedFormulaSuggestion(cell, repaired, evidence)); + analysis.findings.push(focusedFormulaFinding(cell, repaired, evidence)); + } + + const bridgeRows = cells.flatMap((cell) => { + if (!cell.formula) return []; + const label = displayValue(nearestLeftLabel(cell, cellsByKey)?.value); + const parsed = parseCrossSheetDifferenceFormula(cell.formula); + return parsed && /\b(?:growth|expansion)\b/i.test(label) ? [{ cell, label, parsed }] : []; + }); + for (const growth of bridgeRows.filter((entry) => /\bgrowth\b/i.test(entry.label))) { + const growthPosition = parseAddress(growth.cell.address)!; + const peer = bridgeRows.find((entry) => { + const position = parseAddress(entry.cell.address)!; + return entry.cell.sheet === growth.cell.sheet + && /\bexpansion\b/i.test(entry.label) + && position.col === growthPosition.col + && position.row > growthPosition.row + && position.row <= growthPosition.row + 4 + && entry.parsed.sheet === growth.parsed.sheet + && entry.parsed.lower.col === growth.parsed.lower.col; + }); + if (!peer || peer.parsed.upper.col === growth.parsed.upper.col + || peer.parsed.upper.row !== growth.parsed.upper.row + 1) continue; + const repairedAddress = preserveAddressAnchors(growth.parsed.upper.raw, peer.parsed.upper.col, growth.parsed.upper.row); + const repaired = growth.cell.formula!.replace(growth.parsed.upper.raw, repairedAddress); + const evidence = [`${growth.cell.address} and ${peer.cell.address} are paired growth/expansion bridges with the same baseline; ${peer.parsed.upper.raw} establishes the exit-period column`]; + analysis.formulaSuggestions.push(focusedFormulaSuggestion(growth.cell, repaired, evidence)); + analysis.findings.push(focusedFormulaFinding(growth.cell, repaired, evidence)); + } + + for (const cell of cells) { + const text = typeof unwrapCellValue(cell.value) === "string" ? displayValue(cell.value).trim().toLowerCase() : ""; + if (!/^(?:the|and|or)$/.test(text) || cell.formula) continue; + const neighbors = cellNeighbors(cell.sheet, cell.address, cellsByKey, 1); + if (neighbors.some((neighbor) => neighbor.formula || !isBlank(unwrapCellValue(neighbor.value)))) continue; + const evidence = [`${cell.address} contains the isolated stopword ${JSON.stringify(text)} in an otherwise blank local region`]; + analysis.valueSuggestions.push({ confidence: "high", sheet: cell.sheet, cell: cell.address, value: "", evidence }); + analysis.findings.push({ + kind: "hardcoded_in_formula_band", + severity: "error", + sheet: cell.sheet, + address: cell.address, + detail: evidence[0], + recommendedAction: `Clear only ${cell.address}.`, + }); + } + return analysis; +} + +function parseCrossSheetDifferenceFormula(formula: string): { + sheet: string; + upper: { raw: string; col: number; row: number }; + lower: { raw: string; col: number; row: number }; +} | undefined { + const match = normalizeFormula(formula)?.match(/^\((?:'((?:[^']|'')+)'|([A-Za-z_][A-Za-z0-9_. -]*))!(\$?[A-Z]{1,3}\$?[1-9][0-9]*)-(?:'\1'|\2)!(\$?[A-Z]{1,3}\$?[1-9][0-9]*)\)\*/i); + if (!match) return undefined; + const upper = parseAddress(match[3]); + const lower = parseAddress(match[4]); + return upper && lower ? { + sheet: (match[1] ?? match[2]).replace(/''/g, "'").trim(), + upper: { raw: match[3], ...upper }, + lower: { raw: match[4], ...lower }, + } : undefined; +} + +function analyzeSignConventionAudit(cells: WorkbookObservedCell[]): FocusedAuditAnalysis { + const analysis = emptyFocusedAuditAnalysis(); + const cellsByKey = new Map(cells.map((cell) => [workbookCellKey(cell.sheet, cell.address), cell])); + for (const cell of cells.filter((candidate) => !!candidate.formula)) { + const terms = parseSimpleAdditiveFormula(cell.formula!); + if (!terms || terms.length < 2) continue; + let explicitLabelCount = 0; + const repairedTerms = terms.map((term, index) => { + const source = cellsByKey.get(workbookCellKey(cell.sheet, term.address)); + const label = source ? nearestLeftLabel(source, cellsByKey) : undefined; + const sign = displayValue(label?.value).match(/^\s*\(\s*([+-])\s*\)/)?.[1] as "+" | "-" | undefined; + if (sign) explicitLabelCount += 1; + return { ...term, sign: sign ?? (index === 0 && !term.sign ? "+" : term.sign || "+") }; + }); + let repaired = repairedTerms.map((term) => `${term.sign}${term.rawAddress}`).join(""); + const evidence: string[] = []; + if (explicitLabelCount > 0 && formulaWithoutLeadingPlus(repaired) !== formulaWithoutLeadingPlus(cell.formula!)) { + evidence.push(`${explicitLabelCount} referenced row label(s) explicitly declare (+) or (-) treatment`); + } else { + repaired = cell.formula!; + } + + if (normalizeFormula(repaired) === normalizeFormula(cell.formula) && terms.length === 2) { + const targetPosition = parseAddress(cell.address)!; + const totalLabel = nearestLeftLabel(cellsByKey.get(workbookCellKey(cell.sheet, terms[0].address))!, cellsByKey); + for (let offset = 1; offset <= 2; offset += 1) { + const balancingCell = cellsByKey.get(workbookCellKey(cell.sheet, addressFromPosition(targetPosition.row + offset, targetPosition.col))); + if (!balancingCell?.formula) continue; + const balancingTerms = parseSimpleAdditiveFormula(balancingCell.formula); + const balancingLabel = balancingCell ? nearestLeftLabel(balancingCell, cellsByKey) : undefined; + if (!balancingLabel || !totalLabel) continue; + const balancingReferences = balancingTerms?.map((term) => term.address) + ?? formulaCellReferences(balancingCell.formula); + const referencesTarget = balancingReferences.some((address) => normalizeAddress(address) === normalizeAddress(cell.address)); + const referencesComponent = balancingReferences.some((address) => normalizeAddress(address) === normalizeAddress(terms[1].address)); + const sameTotalLabel = normalizeSemanticLabel(displayValue(balancingLabel.value)) === normalizeSemanticLabel(displayValue(totalLabel.value)); + if (!referencesTarget || !referencesComponent || !sameTotalLabel || !/total/i.test(displayValue(totalLabel.value))) continue; + repaired = `+${terms[0].rawAddress}-${terms[1].rawAddress}`; + evidence.push(`${balancingCell.address} recombines ${cell.address} with ${terms[1].address} to reproduce ${displayValue(totalLabel.value)}`); + break; + } + } + + if (normalizeFormula(repaired) === normalizeFormula(cell.formula)) continue; + analysis.formulaSuggestions.push(focusedFormulaSuggestion(cell, repaired, evidence)); + analysis.findings.push(focusedFormulaFinding(cell, repaired, evidence)); + } + return analysis; +} + +function formulaWithoutLeadingPlus(formula: string): string { + return (normalizeFormula(formula) ?? "").replace(/^\+/, ""); +} + +function analyzeEmbeddedHardcodeAudit(cells: WorkbookObservedCell[]): FocusedAuditAnalysis { + const analysis = emptyFocusedAuditAnalysis(); + const cellsByKey = new Map(cells.map((cell) => [workbookCellKey(cell.sheet, cell.address), cell])); + const bySheet = groupByMap(cells.filter((cell) => !!parseAddress(cell.address)), (cell) => cell.sheet); + for (const sheetCells of bySheet.values()) { + const rows = groupByMap(sheetCells, (cell) => parseAddress(cell.address)!.row); + const formulaRuns = [...rows.values()].flatMap((rowCells) => contiguousCellRuns( + rowCells.filter((cell) => !!cell.formula), + "horizontal", + )).filter((run) => run.length >= 3); + const scalarRuns = [...rows.values()].flatMap((rowCells) => contiguousCellRuns( + rowCells.filter((cell) => !cell.formula && typeof unwrapCellValue(cell.value) === "number"), + "horizontal", + )).filter((run) => run.length >= 3); + for (const targetRun of scalarRuns) { + if (targetRun.some((cell) => normalizeSpreadsheetFontColor(cell.fontColor) === "FF0000FF")) continue; + const targetLabel = nearestLeftLabel(targetRun[0], cellsByKey); + const targetTokens = new Set(semanticLabelTokens(displayValue(targetLabel?.value))); + if (targetTokens.size < 2) continue; + const targetKeys = new Set(targetRun.map((cell) => workbookCellKey(cell.sheet, cell.address))); + const candidates = formulaRuns.flatMap((sourceRun) => { + if (sourceRun.length !== targetRun.length) return []; + const valuesMatch = sourceRun.every((source, index) => focusedValuesEquivalent(unwrapCellValue(source.value), unwrapCellValue(targetRun[index].value))); + const sourceLabel = nearestLeftLabel(sourceRun[0], cellsByKey); + const sourceTokens = new Set(semanticLabelTokens(displayValue(sourceLabel?.value))); + const overlap = [...sourceTokens].filter((token) => targetTokens.has(token)).length; + const specificity = semanticLabelSpecificityScore(targetTokens, sourceTokens); + const formatMatches = workbookFormatKind(sourceRun[0]) === workbookFormatKind(targetRun[0]); + const dependsOnTarget = sourceRun.some((source) => formulaTransitivelyDependsOnTargets( + source, + targetKeys, + cellsByKey, + )); + return overlap >= 2 && formatMatches && !dependsOnTarget ? [{ sourceRun, overlap, specificity, valuesMatch }] : []; + }).sort((left, right) => Number(right.valuesMatch) - Number(left.valuesMatch) + || right.specificity - left.specificity + || right.overlap - left.overlap); + if (!candidates[0] || (candidates[1] + && candidates[1].valuesMatch === candidates[0].valuesMatch + && candidates[1].specificity === candidates[0].specificity + && candidates[1].overlap === candidates[0].overlap)) continue; + targetRun.forEach((target, index) => { + const source = candidates[0].sourceRun[index]; + const formula = `+${source.address}`; + const evidence = [`${targetRun[0].address}:${targetRun.at(-1)!.address} duplicates the formula-backed ${sourceRunRange(candidates[0].sourceRun)} values under matching row labels`]; + analysis.formulaSuggestions.push(focusedFormulaSuggestion(target, formula, evidence, "fill_gap")); + analysis.findings.push(focusedFormulaFinding(target, formula, evidence, "hardcoded_in_formula_band")); + }); + } + + for (const targetRun of scalarRuns.filter((run) => run.length >= 3)) { + const label = displayValue(nearestLeftLabel(targetRun[0], cellsByKey)?.value); + if (!/\bexit\b.*\bmultiple\b/i.test(label)) continue; + const labeledControls = sheetCells.flatMap((cell) => { + if (!cell.formula || !parseAddress(cell.address)) return []; + const controlLabel = displayValue(nearestLeftLabel(cell, cellsByKey)?.value); + return /^\s*(?:entry|exit)\b.*\bmultiple\b/i.test(controlLabel) ? [{ cell, controlLabel }] : []; + }); + const entry = labeledControls.filter(({ controlLabel }) => /^\s*entry\b/i.test(controlLabel)); + const entryPosition = entry.length === 1 ? parseAddress(entry[0].cell.address) : undefined; + const exit = labeledControls.filter(({ cell, controlLabel }) => { + const position = parseAddress(cell.address); + return /^\s*exit\b/i.test(controlLabel) && (!entryPosition || position?.col === entryPosition.col); + }); + if (entry.length !== 1 || exit.length !== 1) continue; + for (const target of targetRun) { + const position = parseAddress(target.address)!; + const header = cellsByKey.get(workbookCellKey(target.sheet, addressFromPosition(position.row - 2, position.col))); + const isActual = /&\s*"A"/i.test(header?.formula ?? ""); + const isProjected = /&\s*"P"/i.test(header?.formula ?? ""); + if (!isActual && !isProjected) continue; + const source = isActual ? entry[0].cell : exit[0].cell; + const sourcePosition = parseAddress(source.address)!; + const formula = isActual ? `+${source.address}` : `$${columnNumberToName(sourcePosition.col)}$${sourcePosition.row}`; + const evidence = [`${target.address} is a hardcoded ${isActual ? "actual" : "projected"} exit multiple; ${source.address} is the uniquely labeled ${isActual ? "entry" : "exit"} multiple control`]; + analysis.formulaSuggestions.push(focusedFormulaSuggestion(target, formula, evidence, "fill_gap")); + analysis.findings.push(focusedFormulaFinding(target, formula, evidence, "hardcoded_in_formula_band")); + } + } + } + + const formulaBandsByCell = new Map(); + const formulaRows = groupByMap(cells.filter((cell) => !!cell.formula && !!parseAddress(cell.address)), (cell) => { + const position = parseAddress(cell.address)!; + return `${cell.sheet}\u0000${position.row}`; + }); + for (const rowCells of formulaRows.values()) { + for (const run of contiguousCellRuns(rowCells, "horizontal").filter((candidate) => candidate.length >= 3)) { + for (const cell of run) formulaBandsByCell.set(workbookCellKey(cell.sheet, cell.address), run); + } + } + + for (const cell of cells.filter((candidate) => !!candidate.formula)) { + const normalized = normalizeFormula(cell.formula)!; + const differenceLiteral = normalized.match(/^\(((?:'[^']+'|[A-Za-z_][A-Za-z0-9_. -]*)!\$?[A-Z]{1,3}\$?[1-9][0-9]*)-((?:'[^']+'|[A-Za-z_][A-Za-z0-9_. -]*)!\$?[A-Z]{1,3}\$?[1-9][0-9]*)\)\*(-?\d+(?:\.\d+)?)$/i); + if (differenceLiteral && Math.abs(Number(differenceLiteral[3])) > 1) { + const formula = normalized.replace(differenceLiteral[3], differenceLiteral[1]); + const evidence = [`${cell.address} multiplies a cross-sheet bridge by literal ${differenceLiteral[3]}; the upper endpoint ${differenceLiteral[1]} is the visible exit assumption`]; + analysis.formulaSuggestions.push(focusedFormulaSuggestion(cell, formula, evidence)); + analysis.findings.push(focusedFormulaFinding(cell, formula, evidence)); + continue; + } + + const numericLiterals = [...normalized.matchAll(/(? ({ raw: match[0], value: Number(match[0]), index: match.index ?? -1 })) + .filter(({ value }) => Number.isFinite(value) && value !== 0 && Math.abs(value) !== 1); + if (numericLiterals.length !== 1) continue; + const literal = numericLiterals[0]; + if (literal.index < 0 || normalized[literal.index + literal.raw.length] === "%") continue; + const formulaBand = formulaBandsByCell.get(workbookCellKey(cell.sheet, cell.address)); + if (!formulaBand || formulaBand.filter((peer) => + numericFormulaLiterals(peer.formula).some((value) => Math.abs(value - literal.value) <= 1e-9)).length < 3) continue; + const targetTokens = new Set(semanticLabelTokens(displayValue(nearestLeftLabel(cell, cellsByKey)?.value))); + if (targetTokens.size < 2) continue; + const candidates = cells.flatMap((candidate) => { + const value = unwrapCellValue(candidate.value); + if (candidate.sheet !== cell.sheet || candidate.formula || typeof value !== "number" || Math.abs(value - literal.value) > 1e-9) return []; + const label = nearestLeftLabel(candidate, cellsByKey); + const sourceTokens = new Set(semanticLabelTokens(displayValue(label?.value))); + const score = semanticControlMatchScore(targetTokens, sourceTokens); + const sourcePosition = parseAddress(candidate.address); + const targetPosition = parseAddress(cell.address); + const distance = sourcePosition && targetPosition + ? Math.abs(sourcePosition.row - targetPosition.row) + Math.abs(sourcePosition.col - targetPosition.col) + : Number.MAX_SAFE_INTEGER; + return score >= 3 ? [{ candidate, score, distance }] : []; + }).sort((left, right) => right.score - left.score || left.distance - right.distance); + if (!candidates[0] || (candidates[1] + && candidates[1].score === candidates[0].score + && candidates[1].distance === candidates[0].distance)) continue; + const source = candidates[0].candidate; + const sourcePosition = parseAddress(source.address)!; + const replacement = `$${columnNumberToName(sourcePosition.col)}$${sourcePosition.row}`; + const formula = normalized.replace(literal.raw, replacement); + const evidence = [`literal ${literal.raw} in ${cell.address} matches the uniquely labeled control ${source.sheet}!${source.address}`]; + analysis.formulaSuggestions.push(focusedFormulaSuggestion(cell, formula, evidence)); + analysis.findings.push(focusedFormulaFinding(cell, formula, evidence)); + } + return analysis; +} + +type FocusedAggregateRange = { + functionName: "AVERAGE" | "MEDIAN" | "MAX" | "MIN"; + match: string; + sourceSheet: string; + sheetToken: string; + startToken: string; + endToken: string; + start: string; + end: string; +}; + +function analyzeIncorrectAverageAudit(cells: WorkbookObservedCell[]): FocusedAuditAnalysis { + const analysis = emptyFocusedAuditAnalysis(); + const cellsByKey = new Map(cells.map((cell) => [workbookCellKey(cell.sheet, cell.address), cell])); + const entries = cells.flatMap((cell) => { + const parsed = cell.formula ? parseFocusedAggregateRange(cell.formula, cell.sheet) : undefined; + const target = parseAddress(cell.address); + const start = parsed ? parseAddress(parsed.start) : undefined; + const end = parsed ? parseAddress(parsed.end) : undefined; + return parsed && target && start && end ? [{ cell, parsed, target, start, end }] : []; + }); + + const periodicGroups = groupByMap(entries.filter(({ start, end }) => start.col === end.col), ({ cell, parsed, target, start }) => + `${cell.sheet}\u0000${target.row}\u0000${parsed.functionName}\u0000${parsed.sourceSheet}\u0000${start.col}`); + for (const group of periodicGroups.values()) { + for (const run of contiguousCellRuns(group.map(({ cell }) => cell), "horizontal")) { + if (run.length < 3) continue; + const members = run.map((cell) => group.find((entry) => entry.cell === cell)!); + const periodLength = members[1].start.row - members[0].start.row; + if (periodLength < 2 || periodLength > 366) continue; + const regularStarts = members.every((entry, index) => index === 0 + || entry.start.row - members[index - 1].start.row === periodLength); + const oneShort = members.every(({ start, end }) => end.row - start.row + 1 === periodLength - 1); + const terminalCells = members.map(({ parsed, end }) => + cellsByKey.get(workbookCellKey(parsed.sourceSheet, addressFromPosition(end.row + 1, end.col)))); + const populatedTerminals = terminalCells.every((terminal) => terminal + && ((terminal.formula && !FORMULA_ERROR_RE.test(terminal.formula)) + || typeof unwrapCellValue(terminal.value) === "number")); + if (!regularStarts || !oneShort || !populatedTerminals) continue; + for (const entry of members) { + const repairedEnd = addressFromPosition(entry.end.row + 1, entry.end.col); + const formula = replaceFocusedAggregateRange(entry.cell.formula!, entry.parsed, entry.parsed.start, repairedEnd); + const evidence = [ + `${sourceRunRange(run)} uses regular ${periodLength}-row source periods, but every ${entry.parsed.functionName} range is one row short`, + `${entry.parsed.sourceSheet}!${repairedEnd} is the populated terminal row immediately before the next period boundary`, + ]; + analysis.formulaSuggestions.push(focusedFormulaSuggestion(entry.cell, formula, evidence)); + analysis.findings.push(focusedFormulaFinding(entry.cell, formula, evidence, "formula_range_anomaly")); + } + } + } + + const panelGroups = groupByMap(entries.filter(({ cell, parsed, start, end }) => + parsed.sourceSheet.toLowerCase() === cell.sheet.toLowerCase() && start.row === end.row), ({ cell, parsed, target, start }) => + `${cell.sheet}\u0000${target.row}\u0000${parsed.functionName}\u0000${parsed.sourceSheet}\u0000${start.row}`); + for (const group of panelGroups.values()) { + if (group.length < 3) continue; + const ordered = [...group].sort((left, right) => left.target.col - right.target.col); + const panelStep = ordered[1].target.col - ordered[0].target.col; + if (panelStep < 2 || !ordered.every((entry, index) => index === 0 + || entry.target.col - ordered[index - 1].target.col === panelStep)) continue; + const counts = new Map(); + for (const entry of ordered) { + const key = `${entry.parsed.start}:${entry.parsed.end}`; + counts.set(key, (counts.get(key) ?? 0) + 1); + } + const ranked = [...counts.entries()].sort((left, right) => right[1] - left[1]); + if (!ranked[0] || ranked[0][1] < 2 || ranked[0][1] === ranked[1]?.[1]) continue; + const [consensusStart, consensusEnd] = ranked[0][0].split(":"); + for (const entry of ordered.filter(({ parsed }) => `${parsed.start}:${parsed.end}` !== ranked[0][0])) { + const targetKeys = new Set([workbookCellKey(entry.cell.sheet, entry.cell.address)]); + const currentRangeCells: WorkbookObservedCell[] = []; + for (let col = entry.start.col; col <= entry.end.col; col += 1) { + const source = cellsByKey.get(workbookCellKey(entry.parsed.sourceSheet, addressFromPosition(entry.start.row, col))); + if (source) currentRangeCells.push(source); + } + if (!currentRangeCells.some((source) => formulaTransitivelyDependsOnTargets(source, targetKeys, cellsByKey))) continue; + const formula = replaceFocusedAggregateRange(entry.cell.formula!, entry.parsed, consensusStart, consensusEnd); + const evidence = [ + `${ordered.filter(({ parsed }) => `${parsed.start}:${parsed.end}` === ranked[0][0]).map(({ cell }) => cell.address).join(" and ")} establish ${consensusStart}:${consensusEnd} across equally spaced scenario panels`, + `${entry.parsed.start}:${entry.parsed.end} includes a formula that depends on ${entry.cell.address}, creating a transitive aggregate cycle`, + ]; + analysis.formulaSuggestions.push(focusedFormulaSuggestion(entry.cell, formula, evidence)); + analysis.findings.push(focusedFormulaFinding(entry.cell, formula, evidence, "formula_range_anomaly")); + } + } + return analysis; +} + +function parseFocusedAggregateRange(formula: string, fallbackSheet: string): FocusedAggregateRange | undefined { + const match = formula.match( + /\b(AVERAGE|MEDIAN|MAX|MIN)\s*\(\s*((?:'(?:[^']|'')+'|[A-Za-z_][A-Za-z0-9_.]*)!\s*)?(\$?[A-Z]{1,3}\$?[1-9][0-9]*)\s*:\s*(\$?[A-Z]{1,3}\$?[1-9][0-9]*)\s*\)/i, + ); + if (!match) return undefined; + const sheetToken = match[2] ?? ""; + const rawSheet = sheetToken.trim().replace(/!$/, ""); + return { + functionName: match[1].toUpperCase() as FocusedAggregateRange["functionName"], + match: match[0], + sourceSheet: rawSheet ? rawSheet.replace(/^'|'$/g, "").replace(/''/g, "'") : fallbackSheet, + sheetToken, + startToken: match[3], + endToken: match[4], + start: normalizeAddress(match[3]), + end: normalizeAddress(match[4]), + }; +} + +function replaceFocusedAggregateRange( + formula: string, + parsed: FocusedAggregateRange, + start: string, + end: string, +): string { + return formula.replace( + parsed.match, + `${parsed.functionName}(${parsed.sheetToken}${anchoredAddress(parsed.startToken, start)}:${anchoredAddress(parsed.endToken, end)})`, + ); +} + +function focusedFormulaSuggestion( + cell: WorkbookObservedCell, + formula: string, + evidence: string[], + kind: "fill_gap" | "replace_outlier" = "replace_outlier", +): WorkbookTaskInspection["formulaRepairSuggestions"][number] { + return { kind, confidence: "high", sheet: cell.sheet, cell: cell.address, formula, evidence }; +} + +function focusedFormulaFinding( + cell: WorkbookObservedCell, + formula: string, + evidence: string[], + kind: WorkbookInspectionFindingKind = "formula_pattern_outlier", +): WorkbookInspectionFinding { + return { + kind, + severity: "error", + sheet: cell.sheet, + address: cell.address, + detail: evidence.join("; "), + recommendedAction: `Replace only ${cell.address} with ${formula} and verify the local invariant.`, + }; +} + +function contiguousCellRuns(cells: WorkbookObservedCell[], direction: "horizontal" | "vertical"): WorkbookObservedCell[][] { + const sorted = [...cells].filter((cell) => !!parseAddress(cell.address)).sort((left, right) => { + const a = parseAddress(left.address)!; + const b = parseAddress(right.address)!; + return direction === "horizontal" ? a.col - b.col : a.row - b.row; + }); + const runs: WorkbookObservedCell[][] = []; + for (const cell of sorted) { + const position = parseAddress(cell.address)!; + const current = runs.at(-1); + const previous = current?.at(-1); + const previousPosition = previous ? parseAddress(previous.address)! : undefined; + const contiguous = previousPosition && (direction === "horizontal" + ? position.row === previousPosition.row && position.col === previousPosition.col + 1 + : position.col === previousPosition.col && position.row === previousPosition.row + 1); + if (!current || !contiguous) runs.push([cell]); + else current.push(cell); + } + return runs; +} + +function preserveAddressAnchors(address: string, col: number, row: number): string { + const match = address.match(/^(\$?)[A-Z]{1,3}(\$?)[1-9][0-9]*$/i); + return `${match?.[1] ?? ""}${columnNumberToName(col)}${match?.[2] ?? ""}${row}`; +} + +function replaceAddressRow(address: string, row: number): string { + const position = parseAddress(address); + return position ? preserveAddressAnchors(address, position.col, row) : address; +} + +function preserveReferenceAnchors(template: string, address: string): string { + const position = parseAddress(address); + return position ? preserveAddressAnchors(template, position.col, position.row) : address; +} + +function sheetNameFromFormulaToken(token: string): string | undefined { + const cleaned = token.trim().replace(/!$/, ""); + if (!cleaned) return undefined; + return cleaned.replace(/^'|'$/g, "").replace(/''/g, "'"); +} + +function semanticLabelTokens(value: string): string[] { + const normalized = value.toLowerCase() + .replace(/\badj\.?\b/g, "adjusted") + .replace(/%/g, " percent ") + .replace(/[^a-z0-9]+/g, " "); + const stop = new Set(["the", "and", "of", "as", "to", "in", "entry", "fy", "year", "case", "amdocs", "total"]); + return [...new Set((normalized.match(/[a-z][a-z0-9]*/g) ?? []).filter((token) => !stop.has(token) && !/^20\d{2}$/.test(token)))]; +} + +function focusedValuesEquivalent(left: unknown, right: unknown): boolean { + if (typeof left === "number" && typeof right === "number") { + return Math.abs(left - right) <= Math.max(1e-9, Math.abs(left) * 1e-9, Math.abs(right) * 1e-9); + } + return displayValue(left) === displayValue(right); +} + +function formulaCellReferences(formula: string): string[] { + return [...formula.matchAll(CELL_TOKEN_RE)].map((match) => match[0]); +} + +function formulaTransitivelyDependsOnTargets( + cell: WorkbookObservedCell, + targetKeys: Set, + cellsByKey: Map, + seen = new Set(), + depth = 0, +): boolean { + if (!cell.formula || depth > 24) return false; + const key = workbookCellKey(cell.sheet, cell.address); + if (seen.has(key)) return false; + seen.add(key); + for (const reference of formulaCellReferences(cell.formula)) { + const referenceKey = workbookCellKey(cell.sheet, reference); + if (targetKeys.has(referenceKey)) return true; + const dependency = cellsByKey.get(referenceKey); + if (dependency && formulaTransitivelyDependsOnTargets(dependency, targetKeys, cellsByKey, seen, depth + 1)) return true; + } + return false; +} + +function numericFormulaLiterals(formula: string | undefined): number[] { + const normalized = normalizeFormula(formula) ?? ""; + return [...normalized.matchAll(/(? normalized[(match.index ?? -1) + match[0].length] !== "%") + .map((match) => Number(match[0])) + .filter((value) => Number.isFinite(value) && value !== 0 && Math.abs(value) !== 1); +} + +function semanticControlMatchScore(targetTokens: Set, sourceTokens: Set): number { + let score = [...sourceTokens].filter((token) => targetTokens.has(token)).length; + if (targetTokens.has("interest") && sourceTokens.has("rate")) score += 1; + if (sourceTokens.has("debt") && ["debt", "secured", "tlb"].some((token) => targetTokens.has(token))) score += 1; + return score; +} + +function semanticLabelSpecificityScore(targetTokens: Set, sourceTokens: Set): number { + const generic = new Set(["adjusted", "revenue", "percent", "amount", "value"]); + return [...sourceTokens] + .filter((token) => targetTokens.has(token)) + .reduce((score, token) => score + (generic.has(token) ? 1 : 2), 0); +} + +function replaceNthCellReference(formula: string, targetIndex: number, replacement: string): string { + let index = 0; + return formula.replace(CELL_TOKEN_RE, (match) => index++ === targetIndex ? replacement : match); +} + +function nearestLeftLabel( + cell: WorkbookObservedCell | undefined, + cellsByKey: Map, +): WorkbookObservedCell | undefined { + if (!cell) return undefined; + const position = parseAddress(cell.address); + if (!position) return undefined; + for (let col = position.col - 1; col >= Math.max(1, position.col - 12); col -= 1) { + const candidate = cellsByKey.get(workbookCellKey(cell.sheet, addressFromPosition(position.row, col))); + if (typeof unwrapCellValue(candidate?.value) === "string" && displayValue(unwrapCellValue(candidate?.value)).trim()) return candidate; + } + return undefined; +} + +function workbookFormatKind(cell: WorkbookObservedCell): "percent" | "currency" | "multiple" | "number" { + const format = cell.numFmt ?? ""; + if (/%/.test(format)) return "percent"; + if (/[$£€¥]/.test(format)) return "currency"; + if (/0\.0x|\bx\b/i.test(format)) return "multiple"; + return "number"; +} + +type SimpleCrossSheetReference = { sheet: string; address: string; addressText: string }; + +function parseSimpleCrossSheetReference(formula: string): SimpleCrossSheetReference | undefined { + const match = normalizeFormula(formula)?.match(/^\+?(?:'((?:[^']|'')+)'|([A-Za-z_][A-Za-z0-9_. -]*))!(\$?[A-Z]{1,3}\$?[1-9][0-9]*)$/i); + if (!match) return undefined; + return { + sheet: (match[1] ?? match[2]).replace(/''/g, "'").trim(), + address: normalizeAddress(match[3]), + addressText: match[3], + }; +} + +function visibleYearKey(value: unknown): number | undefined { + if (typeof value === "number" && Number.isInteger(value) && value >= 1900 && value <= 2200) return value; + const match = displayValue(value).match(/\b(20\d{2})\b/); + return match ? Number(match[1]) : undefined; +} + +function topmostColumnYear( + cellsByKey: Map, + sheet: string, + col: number, +): number | undefined { + for (let row = 1; row <= 64; row += 1) { + const year = resolvedVisibleYear(cellsByKey, sheet, addressFromPosition(row, col)); + if (year) return year; + } + return undefined; +} + +function resolvedVisibleYear( + cellsByKey: Map, + sheet: string, + address: string, + seen = new Set(), +): number | undefined { + const key = workbookCellKey(sheet, address); + if (seen.has(key) || seen.size > 24) return undefined; + seen.add(key); + const cell = cellsByKey.get(key); + const direct = visibleYearKey(unwrapCellValue(cell?.value)); + if (direct) return direct; + const predecessor = normalizeFormula(cell?.formula)?.match(/^\+?(\$?[A-Z]{1,3}\$?[1-9][0-9]*)\+1$/i)?.[1]; + if (!predecessor) return undefined; + const base = resolvedVisibleYear(cellsByKey, sheet, predecessor, seen); + return base ? base + 1 : undefined; +} + +type SimpleAdditiveTerm = { sign: "+" | "-" | ""; address: string; rawAddress: string }; + +function parseSimpleAdditiveFormula(formula: string): SimpleAdditiveTerm[] | undefined { + const normalized = normalizeFormula(formula); + if (!normalized || !/^\+?\$?[A-Z]{1,3}\$?[1-9][0-9]*(?:[+-]\$?[A-Z]{1,3}\$?[1-9][0-9]*)+$/i.test(normalized)) return undefined; + return [...normalized.matchAll(/([+-]?)(\$?[A-Z]{1,3}\$?[1-9][0-9]*)/gi)].map((match) => ({ + sign: (match[1] ?? "") as "+" | "-" | "", + address: normalizeAddress(match[2]), + rawAddress: match[2], + })); +} + +function sourceRunRange(run: WorkbookObservedCell[]): string { + return `${run[0].address}:${run.at(-1)!.address}`; +} + +function auditFocusAllowsFormulaSuggestion( + focus: WorkbookAuditFocus, + suggestion: WorkbookTaskInspection["formulaRepairSuggestions"][number], + cell: WorkbookObservedCell | undefined, +): boolean { + const actual = normalizeFormula(cell?.formula); + const proposed = normalizeFormula(suggestion.formula); + if (!proposed) return false; + if (focus === "embedded_hardcode") { + return suggestion.kind === "fill_gap" && !!cell && !cell.formula && !isBlank(unwrapCellValue(cell.value)); + } + if (suggestion.kind !== "replace_outlier" || !actual || actual === proposed) return false; + if (focus === "incorrect_average") return /\bAVERAGE\s*\(/i.test(actual) && /\bAVERAGE\s*\(/i.test(proposed); + // A clean downstream formula can cache #REF! solely because one of its + // dependencies is broken. Replacing that formula from horizontal peers + // corrupts semantic columns such as annual totals and YTD aggregates. Only + // authorize an outlier replacement when the formula text itself is broken. + if (focus === "formula_errors") return FORMULA_ERROR_RE.test(actual); + if (focus === "double_counting") return formulasDifferByRangeEndpoint(actual, proposed); + if (focus === "index_match") return /\b(?:INDEX|MATCH)\s*\(/i.test(actual) && /\b(?:INDEX|MATCH)\s*\(/i.test(proposed); + if (focus === "cross_sheet_reference") return formulaSheetReferences(actual).join("\u0000") !== formulaSheetReferences(proposed).join("\u0000") + && (formulaSheetReferences(actual).length > 0 || formulaSheetReferences(proposed).length > 0); + if (focus === "unit_mismatch") return /(?:\*|\/)(?:1,?000|1,?000,?000|10\^\d+|1e[36])/i.test(`${actual}${proposed}`); + if (focus === "sign_convention") return formulaWithoutSigns(actual) === formulaWithoutSigns(proposed); + if (focus === "relative_absolute_reference") return actual.replace(/\$/g, "") === proposed.replace(/\$/g, ""); + return false; +} + +function auditFocusAllowsFinding( + focus: WorkbookAuditFocus, + finding: WorkbookInspectionFinding, + focusTargetKeys: Set, +): boolean { + const key = workbookCellKey(finding.sheet, finding.address); + if (focusTargetKeys.has(key)) return true; + if (focus === "formula_errors") return finding.kind === "formula_error" || finding.kind === "formula_self_reference"; + if (focus === "embedded_hardcode") return finding.kind === "hardcoded_in_formula_band"; + if (focus === "color_coding") return finding.kind === "font_color_anomaly"; + return false; +} + +function formulasDifferByRangeEndpoint(left: string, right: string): boolean { + if (!/\bSUM\s*\(/i.test(left) || !/\bSUM\s*\(/i.test(right)) return false; + const range = /(?:(?:'[^']+'|[A-Za-z0-9_. -]+)!\s*)?(\$?[A-Z]{1,3}\$?[1-9][0-9]*):(\$?[A-Z]{1,3}\$?[1-9][0-9]*)/g; + const leftRanges = [...left.matchAll(range)].map((match) => [normalizeAddress(match[1]), normalizeAddress(match[2])] as const); + const rightRanges = [...right.matchAll(range)].map((match) => [normalizeAddress(match[1]), normalizeAddress(match[2])] as const); + return leftRanges.some((leftRange, index) => { + const rightRange = rightRanges[index]; + return !!rightRange && ( + leftRange[0] === rightRange[0] && leftRange[1] !== rightRange[1] + || leftRange[0] !== rightRange[0] && leftRange[1] === rightRange[1] + ); + }); +} + +function formulaSheetReferences(formula: string): string[] { + return [...formula.matchAll(/(?:'([^']+)'|([A-Za-z0-9_. -]+))!/g)] + .map((match) => (match[1] ?? match[2]).trim().toLowerCase()) + .sort(); +} + +function formulaWithoutSigns(formula: string): string { + return formula + .replace(/\*-?1(?:\.0+)?\b/g, "") + .replace(/\/-?1(?:\.0+)?\b/g, "") + .replace(/[+-]/g, "") + .replace(/\$/g, ""); +} + +type WorkbookFontColorRole = + | "external_direct_formula" + | "external_derived_formula" + | "internal_direct_formula" + | "calculation_formula" + | "hardcoded_input"; + +type WorkbookFontColorEntry = { + cell: WorkbookObservedCell; + role: WorkbookFontColorRole; + family: "external_formula" | "internal_link" | "calculation" | "hardcode"; + position: CellPosition; + color: string; +}; + +function analyzeFontColorAudit(cells: WorkbookObservedCell[]): { + findings: WorkbookInspectionFinding[]; + suggestions: WorkbookTaskInspection["styleSuggestions"]; +} { + const eligible: WorkbookFontColorEntry[] = cells.flatMap((cell) => { + const role = workbookFontColorRole(cell); + const position = parseAddress(cell.address); + return role && position ? [{ + cell, + role, + family: workbookFontColorFamily(role), + position, + color: observedFontColor(cell), + }] : []; + }); + const cellsByKey = new Map(cells.map((cell) => [workbookCellKey(cell.sheet, cell.address), cell])); + const eligibleOrder = new Map(eligible.map((entry, index) => [workbookCellKey(entry.cell.sheet, entry.cell.address), index])); + const eligibleByPosition = new Map(eligible.map((entry) => [fontColorPositionKey(entry.cell.sheet, entry.position), entry])); + const eligibleBySheet = groupByMap(eligible, (entry) => entry.cell.sheet.toLowerCase()); + const hardcodes = eligible.filter((entry) => entry.role === "hardcoded_input"); + const externalDirect = eligible.filter((entry) => entry.role === "external_direct_formula"); + const familyColorCounts = countFontColorEntries(eligible, (entry) => `${entry.family}\u0000${entry.color}`); + const sheetFamilyColorCounts = countFontColorEntries(eligible, (entry) => `${entry.cell.sheet.toLowerCase()}\u0000${entry.family}\u0000${entry.color}`); + const sheetRoleColorCounts = countFontColorEntries(eligible, (entry) => `${entry.cell.sheet.toLowerCase()}\u0000${entry.role}\u0000${entry.color}`); + const strongestFamilyCache = new Map>(); + const familyColorScoreCache = new Map>(); + const roleColorGroups = groupByMap(eligible, (entry) => `${entry.cell.sheet.toLowerCase()}\u0000${entry.role}\u0000${entry.color}`); + const componentCache = new Map(); + const strongestFamilyForSheet = ( + sheet: string, + family: WorkbookFontColorEntry["family"], + ): ReturnType => { + const key = `${sheet.toLowerCase()}\u0000${family}`; + if (!strongestFamilyCache.has(key)) { + strongestFamilyCache.set(key, strongestFamilyColor(eligibleBySheet.get(sheet.toLowerCase()) ?? [], family)); + } + return strongestFamilyCache.get(key); + }; + const familyColorScoreForSheet = ( + sheet: string, + family: WorkbookFontColorEntry["family"], + color: string, + ): ReturnType => { + const key = `${sheet.toLowerCase()}\u0000${family}\u0000${color}`; + if (!familyColorScoreCache.has(key)) { + familyColorScoreCache.set(key, familyColorScore(eligibleBySheet.get(sheet.toLowerCase()) ?? [], family, color)); + } + return familyColorScoreCache.get(key)!; + }; + const componentsForRoleColor = (key: string): WorkbookFontColorEntry[][] => { + const cached = componentCache.get(key); + if (cached) return cached; + const components = spatialComponents(roleColorGroups.get(key) ?? []); + componentCache.set(key, components); + return components; + }; + const nearbyEntries = ( + entry: WorkbookFontColorEntry, + rowRadius: number, + colRadius: number, + predicate: (candidate: WorkbookFontColorEntry) => boolean, + manhattanRadius?: number, + ): WorkbookFontColorEntry[] => { + const matches: WorkbookFontColorEntry[] = []; + for (let rowOffset = -rowRadius; rowOffset <= rowRadius; rowOffset += 1) { + for (let colOffset = -colRadius; colOffset <= colRadius; colOffset += 1) { + if (manhattanRadius !== undefined && Math.abs(rowOffset) + Math.abs(colOffset) > manhattanRadius) continue; + const candidate = eligibleByPosition.get(fontColorPositionKey(entry.cell.sheet, { + row: entry.position.row + rowOffset, + col: entry.position.col + colOffset, + })); + if (candidate && predicate(candidate)) matches.push(candidate); + } + } + return matches.sort((left, right) => + (eligibleOrder.get(workbookCellKey(left.cell.sheet, left.cell.address)) ?? 0) + - (eligibleOrder.get(workbookCellKey(right.cell.sheet, right.cell.address)) ?? 0)); + }; + const candidates = new Map; + evidence: string[]; + related: Set; + }>(); + const addCandidate = ( + entry: WorkbookFontColorEntry, + expected: string, + method: string, + score: number, + evidence: string, + peers: WorkbookFontColorEntry[] = [], + ) => { + if (entry.color === expected || !normalizeSpreadsheetFontColor(expected)) return; + const key = workbookCellKey(entry.cell.sheet, entry.cell.address); + const existing = candidates.get(key); + if (existing && existing.expected !== expected) { + if (existing.score >= score) return; + candidates.delete(key); + } + const candidate = candidates.get(key) ?? { + entry, + expected, + score: 0, + methods: new Set(), + evidence: [], + related: new Set(), + }; + candidate.score += score; + candidate.methods.add(method); + if (!candidate.evidence.includes(evidence)) candidate.evidence.push(evidence); + for (const peer of peers) candidate.related.add(peer.cell.address); + candidates.set(key, candidate); + }; + + // A role majority is accepted only inside one sheet and only with nearby + // corroboration. This preserves intentional color dialects between sheets. + for (const entries of groupByMap(eligible, (entry) => `${entry.cell.sheet.toLowerCase()}\u0000${entry.role}`).values()) { + if (entries[0]?.role === "internal_direct_formula") continue; + const majority = colorMajority(entries); + const hardcodes = entries[0]?.role === "hardcoded_input"; + const minimumCount = hardcodes ? 3 : 2; + const minimumShare = hardcodes ? 0.78 : 2 / 3; + if (!majority || majority.count < minimumCount || majority.count / entries.length < minimumShare) continue; + const componentSizes = spatialComponentSizes(entries); + for (const entry of entries) { + if (entry.color === majority.color) continue; + if (entry.role === "hardcoded_input" && entry.color !== "FF000000") continue; + if (!hardcodes && entry.color === "FFFFFFFF") continue; + if (!hardcodes && (componentSizes.get(workbookCellKey(entry.cell.sheet, entry.cell.address)) ?? 1) > 2) continue; + if (!hardcodes) { + const sameColorInputs = nearbyEntries(entry, 0, 4, (peer) => peer.cell.sheet === entry.cell.sheet + && peer.role === "hardcoded_input" + && peer.color === entry.color + && peer.position.row === entry.position.row); + if (sameColorInputs.length >= 2) continue; + } + const entryShape = relativeFormulaShape(entry.cell); + const entryLabel = normalizedNearestLeftLabel(entry.cell, cellsByKey); + const peers = entries.filter((peer) => { + if (peer.color !== majority.color || peer.cell.address === entry.cell.address) return false; + if (hardcodes) return manhattanDistance(peer.position, entry.position) <= 8; + const translatedPeer = peer.position.row === entry.position.row + && !!entryShape + && relativeFormulaShape(peer.cell) === entryShape; + const mirroredLabel = peer.position.row === entry.position.row + && !!entryLabel + && normalizedNearestLeftLabel(peer.cell, cellsByKey) === entryLabel + && formulaTopology(peer.cell.formula) === formulaTopology(entry.cell.formula); + const compactRoleBand = entries.length <= 5 + && (peer.position.row === entry.position.row || peer.position.col === entry.position.col) + && manhattanDistance(peer.position, entry.position) <= 2; + return translatedPeer || mirroredLabel || compactRoleBand; + }); + if (peers.length === 0) continue; + addCandidate( + entry, + majority.color, + "sheet_role_consensus", + 80 + peers.length, + `${majority.count}/${entries.length} ${entry.role.replace(/_/g, " ")} cells on this sheet use ${majority.color}; ${peers.length} are nearby`, + peers.slice(0, 6), + ); + } + } + + // Learn workbook-wide direct-link conventions, but require a matching local + // peer so neutral output links remain untouched. + for (const role of ["external_direct_formula"] as const) { + const entries = externalDirect; + const majority = colorMajority(entries); + if (!majority || majority.count < 20 || majority.count / entries.length < 0.9) continue; + for (const entry of entries.filter((candidate) => candidate.color !== majority.color)) { + const peers = nearbyEntries(entry, 3, 3, (peer) => peer.cell.sheet === entry.cell.sheet + && peer.role === role + && peer.color === majority.color + && manhattanDistance(peer.position, entry.position) <= 3, 3); + if (peers.length === 0) continue; + addCandidate( + entry, + majority.color, + "workbook_link_convention", + 76 + peers.length, + `${majority.count}/${entries.length} workbook ${role.replace(/_/g, " ")} cells use ${majority.color}, with a matching local peer`, + peers.slice(0, 6), + ); + } + } + + // A unique external link inside an otherwise neutral calculation panel can + // still be locally wrong even when there are no same-role peers. + for (const entry of externalDirect.filter((candidate) => candidate.color !== "FF000000")) { + const sheetKey = entry.cell.sheet.toLowerCase(); + const sameColorExternal = sheetRoleColorCounts.get(`${sheetKey}\u0000external_direct_formula\u0000${entry.color}`) ?? 0; + if (sameColorExternal > 1) continue; + const currentHardcodes = familyColorCounts.get(`hardcode\u0000${entry.color}`) ?? 0; + const currentExternal = familyColorCounts.get(`external_formula\u0000${entry.color}`) ?? 0; + if (currentHardcodes <= currentExternal) continue; + const neutralPeers = nearbyEntries(entry, 8, 8, (peer) => peer.cell.sheet === entry.cell.sheet + && peer.role !== "hardcoded_input" + && peer.color === "FF000000" + && manhattanDistance(peer.position, entry.position) <= 8, 8); + if (neutralPeers.length < 5) continue; + addCandidate( + entry, + "FF000000", + "isolated_external_link_in_neutral_panel", + 88 + neutralPeers.length, + `${neutralPeers.length} nearby formulas establish a neutral calculation panel around this isolated external link`, + neutralPeers.slice(0, 6), + ); + } + + // Large monochrome components can reveal a pasted wrong convention. The + // expected color is learned from color-to-role affinity in the same sheet. + for (const [roleColorKey] of roleColorGroups) { + for (const component of componentsForRoleColor(roleColorKey)) { + if (component.length < 3) continue; + const role = component[0].role; + const family = component[0].family; + if (family !== "external_formula" && family !== "internal_link") continue; + const bounds = componentBounds(component); + if (role === "external_direct_formula" && bounds.width < 8) continue; + const sheetEntries = eligibleBySheet.get(component[0].cell.sheet.toLowerCase()) ?? []; + if (role === "external_derived_formula") { + const currentHardcodes = sheetEntries.filter((entry) => entry.family === "hardcode" && entry.color === component[0].color).length; + if (currentHardcodes < 3) continue; + } + if (role === "internal_direct_formula") { + const currentExternal = sheetEntries.filter((entry) => entry.family === "external_formula" && entry.color === component[0].color).length; + const currentInternal = sheetEntries.filter((entry) => entry.family === "internal_link" && entry.color === component[0].color).length; + if (currentExternal <= currentInternal) continue; + } + const expected = strongestFamilyForSheet(component[0].cell.sheet, family); + if (!expected || expected.color === component[0].color || expected.support < 3 || expected.purity < 0.5) continue; + const current = familyColorScoreForSheet(component[0].cell.sheet, family, component[0].color); + if (expected.score < Math.max(3, current.score * 1.5)) continue; + if (component[0].color === "FF000000") { + const expectedRoleColorKey = `${component[0].cell.sheet.toLowerCase()}\u0000${role}\u0000${expected.color}`; + const overlapping = componentsForRoleColor(expectedRoleColorKey) + .filter((peer) => peer.length >= 8 && componentColumnOverlap(component, peer) >= 0.8); + if (overlapping.length < 2) continue; + } + for (const entry of component) { + addCandidate( + entry, + expected.color, + "spatial_role_affinity", + 92 + component.length, + `${component.length}-cell ${role.replace(/_/g, " ")} component uses ${entry.color}; ${expected.support} same-family cells establish ${expected.color}`, + nearbyEntries(entry, 16, 16, (peer) => peer.cell.sheet === entry.cell.sheet + && peer.family === family + && peer.color === expected.color + && manhattanDistance(peer.position, entry.position) <= 16, 16).slice(0, 6), + ); + } + } + } + + // Scenario matrices often mix seed values and chained formulas. A dense + // local rectangle with one number format supplies stronger evidence than the + // hardcode/formula distinction alone. + for (const entry of hardcodes) { + const peers = nearbyEntries(entry, 1, 3, (peer) => peer.cell.sheet === entry.cell.sheet + && (peer.cell.numFmt ?? "") === (entry.cell.numFmt ?? "") + && Math.abs(peer.position.row - entry.position.row) <= 1 + && Math.abs(peer.position.col - entry.position.col) <= 3); + const majority = colorMajority(peers); + if (!majority || peers.length < 8 || majority.count / peers.length < 0.75 || majority.color === entry.color) continue; + const distinctRows = new Set(peers.map((peer) => peer.position.row)).size; + const distinctCols = new Set(peers.map((peer) => peer.position.col)).size; + if (distinctRows < 2 || distinctCols < 3) continue; + const sheetKey = entry.cell.sheet.toLowerCase(); + const expectedInputSupport = sheetFamilyColorCounts.get(`${sheetKey}\u0000hardcode\u0000${majority.color}`) ?? 0; + const currentInputSupport = sheetFamilyColorCounts.get(`${sheetKey}\u0000hardcode\u0000${entry.color}`) ?? 0; + if (expectedInputSupport < 3 || expectedInputSupport <= currentInputSupport) continue; + addCandidate( + entry, + majority.color, + "dense_scenario_matrix", + 98 + majority.count, + `${majority.count}/${peers.length} nearby cells in the same formatted scenario matrix use ${majority.color}`, + peers.filter((peer) => peer.color === majority.color).slice(0, 6), + ); + } + + // Repeated assumption labels provide a safe vertical-block signal even when + // their value and percentage number formats differ. + const labeledGroups = new Map(); + for (const entry of eligible) { + const label = nearestLeftLabel(entry.cell, cellsByKey); + const tokens = displayValue(unwrapCellValue(label?.value)).toLowerCase().match(/[a-z][a-z0-9]+/g) ?? []; + for (const token of tokens.filter((value) => /^(?:assumptions?|baseline|scenario|case|toggle)$/.test(value))) { + const key = `${entry.cell.sheet.toLowerCase()}\u0000${entry.position.col}\u0000${token}`; + const group = labeledGroups.get(key) ?? []; + group.push(entry); + labeledGroups.set(key, group); + } + } + for (const entries of labeledGroups.values()) { + const ordered = [...entries].sort((left, right) => left.position.row - right.position.row); + const runs: WorkbookFontColorEntry[][] = []; + for (const entry of ordered) { + const run = runs.at(-1); + if (!run || entry.position.row - run.at(-1)!.position.row > 2) runs.push([entry]); + else run.push(entry); + } + for (const run of runs) { + const majority = colorMajority(run); + if (!majority || run.length < 4 || majority.count < 3 || majority.count / run.length < 0.75) continue; + for (const entry of run.filter((candidate) => candidate.color !== majority.color)) { + addCandidate( + entry, + majority.color, + "labeled_assumption_block", + 104 + majority.count, + `${majority.count}/${run.length} adjacent cells with the same assumption label use ${majority.color}`, + run.filter((peer) => peer.color === majority.color), + ); + } + } + } + + const ranked = [...candidates.values()] + .sort((left, right) => right.score - left.score + || left.entry.cell.sheet.localeCompare(right.entry.cell.sheet) + || compareAddresses(left.entry.cell.address, right.entry.cell.address)); + return { + findings: ranked.map((candidate) => ({ + kind: "font_color_anomaly", + severity: "warning", + sheet: candidate.entry.cell.sheet, + address: candidate.entry.cell.address, + relatedAddresses: [...candidate.related].slice(0, 6), + detail: candidate.evidence.join("; "), + recommendedAction: `Change only the font color to ${candidate.expected}; preserve the cell value, formula, number format, and other font attributes.`, + })), + suggestions: ranked.map((candidate) => ({ + kind: "font_color", + confidence: "high", + sheet: candidate.entry.cell.sheet, + cell: candidate.entry.cell.address, + fontColor: candidate.expected, + evidence: candidate.evidence, + })), + }; +} + +function workbookFontColorRole(cell: WorkbookObservedCell): WorkbookFontColorRole | undefined { + const formula = normalizeFormula(cell.formula); + if (formula) { + const direct = /^\+?(?:(?:'((?:[^']|'')+)'|([A-Za-z_][A-Za-z0-9_. -]*))!)?\$?[A-Z]{1,3}\$?[1-9][0-9]*$/i.test(formula); + const external = formulaSheetReferences(formula).some((sheet) => sheet !== cell.sheet.trim().toLowerCase()); + if (external && direct) return "external_direct_formula"; + if (external && pureExternalAggregateFormula(formula, cell.sheet)) return "external_derived_formula"; + return direct ? "internal_direct_formula" : "calculation_formula"; + } + const value = unwrapCellValue(cell.value); + return typeof value === "number" || typeof value === "boolean" || value instanceof Date ? "hardcoded_input" : undefined; +} + +function pureExternalAggregateFormula(formula: string, currentSheet: string): boolean { + if (!/^\+?(?:AVERAGE|SUM|MEDIAN|MIN|MAX)\s*\([^()]+\)$/i.test(formula)) return false; + const references = formulaSheetReferences(formula); + return references.length > 0 && references.every((sheet) => sheet !== currentSheet.trim().toLowerCase()); +} + +function workbookFontColorFamily(role: WorkbookFontColorRole): WorkbookFontColorEntry["family"] { + if (role === "external_direct_formula" || role === "external_derived_formula") return "external_formula"; + if (role === "internal_direct_formula") return "internal_link"; + if (role === "hardcoded_input") return "hardcode"; + return "calculation"; +} + +function fontColorPositionKey(sheet: string, position: CellPosition): string { + return `${sheet.toLowerCase()}\u0000${position.row}:${position.col}`; +} + +function countFontColorEntries( + entries: WorkbookFontColorEntry[], + keyFor: (entry: WorkbookFontColorEntry) => string, +): Map { + const counts = new Map(); + for (const entry of entries) { + const key = keyFor(entry); + counts.set(key, (counts.get(key) ?? 0) + 1); + } + return counts; +} + +function countColors(entries: WorkbookFontColorEntry[]): Map { + const counts = new Map(); + for (const entry of entries) counts.set(entry.color, (counts.get(entry.color) ?? 0) + 1); + return counts; +} + +function colorMajority(entries: WorkbookFontColorEntry[]): { color: string; count: number } | undefined { + const first = [...countColors(entries)].sort((left, right) => right[1] - left[1] || left[0].localeCompare(right[0]))[0]; + return first ? { color: first[0], count: first[1] } : undefined; +} + +function manhattanDistance(left: CellPosition, right: CellPosition): number { + return Math.abs(left.row - right.row) + Math.abs(left.col - right.col); +} + +function spatialComponents(entries: WorkbookFontColorEntry[]): WorkbookFontColorEntry[][] { + const byPosition = new Map(entries.map((entry) => [`${entry.position.row}:${entry.position.col}`, entry])); + const seen = new Set(); + const components: WorkbookFontColorEntry[][] = []; + for (const entry of entries) { + const origin = `${entry.position.row}:${entry.position.col}`; + if (seen.has(origin)) continue; + const component: WorkbookFontColorEntry[] = []; + const queue = [entry]; + let cursor = 0; + seen.add(origin); + while (cursor < queue.length) { + const current = queue[cursor++]; + component.push(current); + for (const [row, col] of [ + [current.position.row - 1, current.position.col], + [current.position.row + 1, current.position.col], + [current.position.row, current.position.col - 1], + [current.position.row, current.position.col + 1], + ]) { + const key = `${row}:${col}`; + const neighbor = byPosition.get(key); + if (!neighbor || seen.has(key)) continue; + seen.add(key); + queue.push(neighbor); + } + } + components.push(component); + } + return components; +} + +function spatialComponentSizes(entries: WorkbookFontColorEntry[]): Map { + const sizes = new Map(); + for (const colorEntries of groupByMap(entries, (entry) => entry.color).values()) { + for (const component of spatialComponents(colorEntries)) { + for (const entry of component) sizes.set(workbookCellKey(entry.cell.sheet, entry.cell.address), component.length); + } + } + return sizes; +} + +function relativeFormulaShape(cell: WorkbookObservedCell): string | undefined { + const formula = normalizeFormula(cell.formula); + const origin = parseAddress(cell.address); + if (!formula || !origin) return undefined; + return formula.replace(CELL_TOKEN_RE, (token) => { + const match = token.match(/^(\$?)([A-Z]{1,3})(\$?)([1-9][0-9]*)$/i); + const position = parseAddress(token); + if (!match || !position) return token.toUpperCase(); + const col = match[1] ? `$${position.col}` : String(position.col - origin.col); + const row = match[3] ? `$${position.row}` : String(position.row - origin.row); + return `R${row}C${col}`; + }); +} + +function formulaTopology(formula: string | undefined): string | undefined { + const normalized = normalizeFormula(formula)?.toUpperCase(); + if (!normalized) return undefined; + return normalized + .replace(/'(?:[^']|'')+'!/g, "SHEET!") + .replace(/\b[A-Z_][A-Z0-9_. -]*!/g, "SHEET!") + .replace(CELL_TOKEN_RE, "CELL") + .replace(/\b\d+(?:\.\d+)?%?/g, "NUM") + .replace(/\$/g, ""); +} + +function normalizedNearestLeftLabel( + cell: WorkbookObservedCell, + cellsByKey: Map, +): string | undefined { + const value = displayValue(unwrapCellValue(nearestLeftLabel(cell, cellsByKey)?.value)) + .trim() + .toLowerCase() + .replace(/\s+/g, " "); + return value || undefined; +} + +function componentBounds(entries: WorkbookFontColorEntry[]): { minCol: number; maxCol: number; width: number } { + const cols = entries.map((entry) => entry.position.col); + const minCol = Math.min(...cols); + const maxCol = Math.max(...cols); + return { minCol, maxCol, width: maxCol - minCol + 1 }; +} + +function componentColumnOverlap(left: WorkbookFontColorEntry[], right: WorkbookFontColorEntry[]): number { + const a = componentBounds(left); + const b = componentBounds(right); + const overlap = Math.max(0, Math.min(a.maxCol, b.maxCol) - Math.max(a.minCol, b.minCol) + 1); + return overlap / Math.max(1, Math.min(a.width, b.width)); +} + +function familyColorScore( + entries: WorkbookFontColorEntry[], + family: WorkbookFontColorEntry["family"], + color: string, +): { color: string; support: number; purity: number; score: number } { + const colorEntries = entries.filter((entry) => entry.color === color); + const support = colorEntries.filter((entry) => entry.family === family).length; + const purity = support / Math.max(1, colorEntries.length); + return { color, support, purity, score: support * purity }; +} + +function strongestFamilyColor( + entries: WorkbookFontColorEntry[], + family: WorkbookFontColorEntry["family"], +): ReturnType | undefined { + const colors = new Set(entries.map((entry) => entry.color)); + return [...colors] + .map((color) => familyColorScore(entries, family, color)) + .sort((left, right) => right.score - left.score || right.support - left.support || left.color.localeCompare(right.color))[0]; +} + +function observedFontColor(cell: WorkbookObservedCell): string { + return normalizeSpreadsheetFontColor(cell.fontColor) ?? "FF000000"; +} + function genericFormulaAuditTask(instruction: string): boolean { return /\b(?:audit\s+and\s+fix|audit\s+this\s+(?:file|workbook)|fix\s+(?:all\s+)?formula\s+(?:errors|inconsistencies)|repair\s+(?:all\s+)?broken\s+(?:formulas|references))\b/i.test(instruction); } diff --git a/src/shared/spreadsheetFontColor.ts b/src/shared/spreadsheetFontColor.ts new file mode 100644 index 00000000..49e0e0cc --- /dev/null +++ b/src/shared/spreadsheetFontColor.ts @@ -0,0 +1,124 @@ +/** Normalize supported spreadsheet font colors to canonical uppercase AARRGGBB. */ +export function normalizeSpreadsheetFontColor(value: unknown): string | undefined { + if (typeof value !== "string") return undefined; + const hex = value.trim().replace(/^#/, ""); + if (!/^(?:[0-9a-fA-F]{6}|[0-9a-fA-F]{8})$/.test(hex)) return undefined; + return (hex.length === 6 ? `FF${hex}` : hex).toUpperCase(); +} + +export type SpreadsheetFontColorSource = { + argb?: unknown; + theme?: unknown; + tint?: unknown; + indexed?: unknown; +}; + +const THEME_COLOR_SLOTS = [ + "lt1", + "dk1", + "lt2", + "dk2", + "accent1", + "accent2", + "accent3", + "accent4", + "accent5", + "accent6", + "hlink", + "folHlink", +] as const; + +// ECMA-376's legacy indexed palette. Indices 64 and 65 are automatic system +// colors, so they intentionally remain unresolved. +const INDEXED_COLORS = [ + "FF000000", "FFFFFFFF", "FFFF0000", "FF00FF00", "FF0000FF", "FFFFFF00", "FFFF00FF", "FF00FFFF", + "FF000000", "FFFFFFFF", "FFFF0000", "FF00FF00", "FF0000FF", "FFFFFF00", "FFFF00FF", "FF00FFFF", + "FF800000", "FF008000", "FF000080", "FF808000", "FF800080", "FF008080", "FFC0C0C0", "FF808080", + "FF9999FF", "FF993366", "FFFFFFCC", "FFCCFFFF", "FF660066", "FFFF8080", "FF0066CC", "FFCCCCFF", + "FF000080", "FFFF00FF", "FFFFFF00", "FF00FFFF", "FF800080", "FF800000", "FF008080", "FF0000FF", + "FF00CCFF", "FFCCFFFF", "FFCCFFCC", "FFFFFF99", "FF99CCFF", "FFFF99CC", "FFCC99FF", "FFFFCC99", + "FF3366FF", "FF33CCCC", "FF99CC00", "FFFFCC00", "FFFF9900", "FFFF6600", "FF666699", "FF969696", + "FF003366", "FF339966", "FF003300", "FF333300", "FF993300", "FF993366", "FF333399", "FF333333", +] as const; + +const themePaletteCache = new Map>(); + +/** Parse an Office theme's color scheme into the numeric indices used by cell styles. */ +export function spreadsheetThemePalette(themeXml: unknown): ReadonlyMap { + if (typeof themeXml !== "string" || !themeXml.trim()) return new Map(); + const cached = themePaletteCache.get(themeXml); + if (cached) return cached; + const palette = new Map(); + const scheme = themeXml.match(/<(?:[A-Za-z_][\w.-]*:)?clrScheme\b[^>]*>[\s\S]*?<\/(?:[A-Za-z_][\w.-]*:)?clrScheme>/i)?.[0] ?? themeXml; + THEME_COLOR_SLOTS.forEach((slot, index) => { + const block = scheme.match(new RegExp( + `<(?:[A-Za-z_][\\w.-]*:)?${slot}\\b[^>]*>([\\s\\S]*?)<\\/(?:[A-Za-z_][\\w.-]*:)?${slot}>`, + "i", + ))?.[1]; + if (!block) return; + const explicit = block.match(/<(?:[A-Za-z_][\w.-]*:)?srgbClr\b[^>]*\bval=["']([0-9a-f]{6}|[0-9a-f]{8})["']/i)?.[1]; + const system = block.match(/<(?:[A-Za-z_][\w.-]*:)?sysClr\b[^>]*\blastClr=["']([0-9a-f]{6}|[0-9a-f]{8})["']/i)?.[1] + ?? block.match(/<(?:[A-Za-z_][\w.-]*:)?sysClr\b[^>]*\bval=["']([0-9a-f]{6}|[0-9a-f]{8})["']/i)?.[1]; + const color = normalizeSpreadsheetFontColor(explicit ?? system); + if (color) palette.set(index, color); + }); + if (themePaletteCache.size >= 32) themePaletteCache.delete(themePaletteCache.keys().next().value ?? ""); + themePaletteCache.set(themeXml, palette); + return palette; +} + +/** Resolve an ExcelJS/OpenXML color token to its effective canonical ARGB color. */ +export function resolveSpreadsheetFontColor( + color: SpreadsheetFontColorSource | null | undefined, + themeXml?: unknown, +): string | undefined { + if (!color) return undefined; + const explicit = normalizeSpreadsheetFontColor(color.argb); + if (explicit) return applySpreadsheetColorTint(explicit, color.tint); + + const theme = finiteInteger(color.theme); + if (theme !== undefined) { + const resolved = spreadsheetThemePalette(themeXml).get(theme); + return resolved ? applySpreadsheetColorTint(resolved, color.tint) : undefined; + } + + const indexed = finiteInteger(color.indexed); + const resolved = indexed === undefined ? undefined : INDEXED_COLORS[indexed]; + return resolved ? applySpreadsheetColorTint(resolved, color.tint) : undefined; +} + +/** Return the workbook theme slot whose effective color matches an ARGB value. */ +export function spreadsheetThemeIndexForColor(value: unknown, themeXml?: unknown): number | undefined { + const color = normalizeSpreadsheetFontColor(value); + if (!color) return undefined; + for (const [index, themedColor] of spreadsheetThemePalette(themeXml)) { + if (themedColor === color) return index; + } + return undefined; +} + +function finiteInteger(value: unknown): number | undefined { + return typeof value === "number" && Number.isInteger(value) && value >= 0 ? value : undefined; +} + +function applySpreadsheetColorTint(argb: string, value: unknown): string { + const tint = typeof value === "number" && Number.isFinite(value) + ? Math.max(-1, Math.min(1, value)) + : 0; + if (tint === 0) return argb; + const alpha = argb.slice(0, 2); + const channels = [argb.slice(2, 4), argb.slice(4, 6), argb.slice(6, 8)].map((hex) => Number.parseInt(hex, 16)); + const tinted = channels.map((channel) => Math.round(tint < 0 + ? channel * (1 + tint) + : channel * (1 - tint) + 255 * tint)); + return `${alpha}${tinted.map((channel) => channel.toString(16).padStart(2, "0")).join("")}`.toUpperCase(); +} + +/** Convert spreadsheet AARRGGBB to CSS #RRGGBB or #RRGGBBAA. */ +export function spreadsheetFontColorToCss(value: unknown): string | undefined { + const argb = normalizeSpreadsheetFontColor(value); + if (!argb) return undefined; + const alpha = argb.slice(0, 2); + const rgb = argb.slice(2); + return alpha === "FF" ? `#${rgb}` : `#${rgb}${alpha}`; +} diff --git a/src/ui/Chat.tsx b/src/ui/Chat.tsx index b7a5f76c..bad94e9c 100644 --- a/src/ui/Chat.tsx +++ b/src/ui/Chat.tsx @@ -6,7 +6,7 @@ import { Reasoning, ReasoningContent, ReasoningTrigger } from "@/components/ai-e import { MessageResponse } from "@/components/ai-elements/message"; import { Suggestion } from "@/components/ai-elements/suggestion"; import { Tool, ToolContent, ToolHeader, type ToolPart } from "@/components/ai-elements/tool"; -import { useStore, CONVEX_SITE_URL, type AgentJobDetailTelemetry, type AgentModelSelection, type PrivateStreamAccess, type RoomStore } from "../app/store"; +import { useStore, CONVEX_SITE_URL, type AgentJobDetailTelemetry, type AgentJobTelemetry, type AgentModelSelection, type PrivateStreamAccess, type RoomStore } from "../app/store"; import { abortable, parseUploadedFiles, UPLOAD_TIMEOUT_MS } from "../app/uploadedArtifact"; import type { StreamId } from "@convex-dev/persistent-text-streaming"; import { api } from "../../convex/_generated/api"; @@ -35,6 +35,7 @@ import { type VoiceRoomStore, } from "../voice"; import "./chat-scale.css"; +import { openWorkArtifactsReview } from "./workArtifacts/workArtifactsNavigation"; const AGENT_AVATAR_COLOR = "#8F3F27"; const COLORS = ["#8F3F27", "#315DA8", "#2F6B44", "#6D3FB2", "#80631F", "#A34B2E"]; @@ -1210,8 +1211,9 @@ export function groupMessagesByDay * match, and a spoofed prefix on a non-agent author is ignored. */ const AGENT_RUN_CLIENT_MSG_ID_RE = /^(?:pubstream|privstream|final|plan-blocked)-(.+)$/; -export function agentRunIdFor(message: { author: { kind: string }; clientMsgId?: string }): string | null { +export function agentRunIdFor(message: { author: { kind: string }; jobId?: string; clientMsgId?: string }): string | null { if (message.author.kind !== "agent") return null; + if (message.jobId?.trim()) return message.jobId.trim(); const match = AGENT_RUN_CLIENT_MSG_ID_RE.exec(message.clientMsgId ?? ""); return match ? match[1] : null; } @@ -1272,6 +1274,36 @@ export function shouldShowJumpToLatest(distanceFromBottom: number, viewportHeigh return viewportHeight > 0 && distanceFromBottom >= viewportHeight * 2; } +export type AgentJobScopeSummary = { reads: string; writes: string; review: string }; + +export function agentJobScopeSummary(job: AgentJobTelemetry, artifacts: Pick[]): AgentJobScopeSummary { + const titleById = new Map(artifacts.map((artifact) => [artifact.id, artifact.title])); + const targetId = job.request?.targetArtifactId ?? job.artifactId; + const target = (targetId && titleById.get(targetId)) || "Room artifact"; + const referenced = (job.request?.references ?? []) + .map((reference) => reference.title || (reference.id ? titleById.get(reference.id) : undefined)) + .filter((title): title is string => !!title); + const reads = [...new Set(referenced.length ? referenced : [target])].slice(0, 3).join(" + "); + const targetCount = job.request?.allowedElementIds?.length ?? 0; + const writes = job.approvalPolicy === "read_only" + ? "None" + : `${target}${targetCount ? ` · ${targetCount} target${targetCount === 1 ? "" : "s"}` : ""}`; + const review = job.approvalPolicy === "auto_commit_safe" + ? "Safe writes auto-commit" + : job.approvalPolicy === "read_only" + ? "Read only" + : "Changes require review"; + return { reads: reads || target, writes, review }; +} + +export function compactElapsed(startedAt: number | undefined, now = Date.now()): string { + if (!startedAt || now <= startedAt) return "0s"; + const seconds = Math.floor((now - startedAt) / 1_000); + if (seconds < 60) return `${seconds}s`; + const minutes = Math.floor(seconds / 60); + return minutes < 60 ? `${minutes}m ${seconds % 60}s` : `${Math.floor(minutes / 60)}h ${minutes % 60}m`; +} + /** One agent run in the feed. Collapsed = one quiet line ("Run · N steps · view"); expanding * restores every original row (and every pinned testid inside) untouched. A group that mounts * while its run is live starts expanded and STAYS expanded after the run finishes — collapse @@ -1389,10 +1421,11 @@ export function Chat({ roomId, me, channel, variant, agentName, activeArtifactId const longJob = isPrivate ? null : store.lastLongFreeJob(); const longJobAttempts = isPrivate ? [] : store.lastLongFreeJobAttempts(); const longJobDetail = isPrivate ? null : store.lastLongFreeJobDetail(); + const roomArtifacts = store.listArtifacts(roomId); const activeArtifact = useMemo(() => { if (!activeArtifactId || isPrivate) return undefined; - return store.listArtifacts(roomId).find((a) => a.id === activeArtifactId); - }, [activeArtifactId, isPrivate, roomId, store]); + return roomArtifacts.find((a) => a.id === activeArtifactId); + }, [activeArtifactId, isPrivate, roomArtifacts]); const decisionState = isPrivate ? null : buildResearchDecisionState(activeArtifact); // The decision card summarises a completed research run — it must stay visible once research lands // (i.e. after the @nodeagent request is sent, when messages exist). A prior `&& messages.length === 0` @@ -1520,6 +1553,18 @@ export function Chat({ roomId, me, channel, variant, agentName, activeArtifactId const canRetryLongJob = !!longJob && ["failed", "blocked", "cancelled", "paused"].includes(longJob.status); const longJobTerminal = !!longJob && ["completed", "failed", "blocked", "cancelled"].includes(longJob.status); const longJobActive = !!longJob && !longJobTerminal; + const [jobNow, setJobNow] = useState(() => Date.now()); + useEffect(() => { + if (!longJobActive) return; + setJobNow(Date.now()); + const timer = window.setInterval(() => setJobNow(Date.now()), 1_000); + return () => window.clearInterval(timer); + }, [longJob?.id, longJobActive]); + const longJobScope = useMemo(() => longJob ? agentJobScopeSummary(longJob, roomArtifacts) : null, [longJob, roomArtifacts]); + const longJobProposalCount = longJob ? store.listProposals(roomId).filter((proposal) => proposal.jobId === longJob.id).length : 0; + const activeReasoningFrame = longJobDetail?.reasoningFrames.find((frame) => frame.status === "running") + ?? longJobDetail?.reasoningFrames.find((frame) => frame.status === "pending"); + const longJobPhase = activeReasoningFrame?.displayName || activeReasoningFrame?.phase || longJob?.status || "idle"; const agentWorking = thinking || (!isPrivate && longJobActive); const unifiedStreamParts = (!isPrivate ? longJobDetail?.streamParts ?? [] : []) as AgentStreamPart[]; const activeJobClientMsgId = !isPrivate && longJob ? `pubstream-${longJob.id}` : ""; @@ -1812,7 +1857,7 @@ export function Chat({ roomId, me, channel, variant, agentName, activeArtifactId }); }; - const send = (raw?: string, overrideModelSelection?: AgentModelSelection) => { + const send = (raw?: string, overrideModelSelection?: AgentModelSelection, disposition?: "start" | "queue" | "redirect") => { const t = (raw ?? text).trim(); if (!t && refs.length === 0) return; const messageRefs = refs; @@ -1828,7 +1873,13 @@ export function Chat({ roomId, me, channel, variant, agentName, activeArtifactId const modelSelection = composerModelSelection(publicNodeAgentRequest.forceFree, overrideModelSelection); beginThinking(); lastAgentInputRef.current = t; - void store.askAgent({ goal: `${publicNodeAgentRequest.goal}${artifactRefContextSuffix(messageRefs)}`, references: messageRefs, modelSelection, contextArtifactId: activeArtifactId }).catch((e) => { + void store.askAgent({ + goal: `${publicNodeAgentRequest.goal}${artifactRefContextSuffix(messageRefs)}`, + references: messageRefs, + modelSelection, + contextArtifactId: activeArtifactId, + disposition: disposition ?? (longJobActive ? "queue" : "start"), + }).catch((e) => { if (aliveRef.current) { setAgentErr(buildAgentFailureNotice(e, { selection: modelSelection, requestText: t, source: "public", jobId: longJob?.id })); setThinking(false); @@ -2090,11 +2141,18 @@ export function Chat({ roomId, me, channel, variant, agentName, activeArtifactId {isPrivate &&
Only you can read this lane in NodeRoom; requests and room context are sent to the configured model provider
} {!isPrivate && showLongJobChrome && longJob && ( -
+
+
+ + {longJobVisibleError + ? humanAgentFailureText(longJobVisibleError) + : longJobPhase.replace(/_/g, " ").replace(/^./, (value) => value.toUpperCase())} + + {compactElapsed(longJob.createdAt, jobNow)} - {longJobVisibleError ? humanAgentFailureText(longJobVisibleError) : longJob.modelPolicy} - {!longJobVisibleError && latestAttempt ? ` · ${latestAttempt.resolvedModel} · ${shortMs(latestAttempt.ms)}` : ""} + {longJob.modelPolicy} + {latestAttempt ? ` · ${latestAttempt.resolvedModel} · ${shortMs(latestAttempt.ms)}` : ""} {longJob.nextRunAt && longJob.status !== "completed" ? ` · next ${clock(longJob.nextRunAt)}` : ""} {(() => { const bad = ["failed", "blocked"].includes(longJob.status); return ( @@ -2127,6 +2185,17 @@ export function Chat({ roomId, me, channel, variant, agentName, activeArtifactId +
+ {longJobScope && ( +
+ Reads{longJobScope.reads} + Writes{longJobScope.writes} + Review{longJobScope.review} + +
+ )}
)} {!isPrivate && showLongJobChrome && longJob && jobDetailsOpen && ( @@ -2434,11 +2503,13 @@ export function Chat({ roomId, me, channel, variant, agentName, activeArtifactId
{!isPrivate && !slashOpen && ( diff --git a/src/ui/RoomShell.tsx b/src/ui/RoomShell.tsx index 64c2cb59..69796a52 100644 --- a/src/ui/RoomShell.tsx +++ b/src/ui/RoomShell.tsx @@ -29,8 +29,18 @@ import type { Actor, Channel } from "../engine/types"; const AUTO_ACCEPT_PREF_KEY = "noderoom:autoAcceptConsent:v1"; const ACTIVE_ARTIFACT_PREF_PREFIX = "noderoom:activeArtifact:v1:"; +export const PANEL_LAYOUT_PREF_KEY = "noderoom:desktopPanelWidths:v1"; const TOUR_KEY = "noderoom:tour:v1"; const NOTE_PRIORITY = ["Capture Notebook", "Note", "Diligence memo", "Open questions / workplan", "Agent wiki"]; +const PANEL_WIDTHS = { + left: { min: 176, max: 380, default: 232 }, + right: { min: 280, max: 560, default: 340 }, +} as const; +const PANEL_RESIZE_STEP = 16; +const STAGE_WIDTH_FLOOR = 760; +const WORKSPACE_EDGE_ALLOWANCE = 30; +export type PanelLayout = { left: number; stage: number; right: number }; +export type PanelResizeTarget = "left" | "right"; type AccentKey = "terra"; type ReplayPace = "brisk" | "standard" | "cinematic"; const ACCENTS: Record = { @@ -118,6 +128,62 @@ function persistRoomArtifact(roomId: string, artifact: { id: string; kind?: stri } } +export function readPersistedPanelLayout(storage?: Pick): PanelLayout { + const fallback = { left: PANEL_WIDTHS.left.default, stage: 1, right: PANEL_WIDTHS.right.default }; + try { + const target = storage ?? (typeof window !== "undefined" ? window.localStorage : undefined); + if (!target) return fallback; + const raw = target.getItem(PANEL_LAYOUT_PREF_KEY); + if (!raw) return fallback; + const parsed = JSON.parse(raw) as { left?: unknown; right?: unknown }; + return { + left: clampFinite(parsed.left, PANEL_WIDTHS.left.min, PANEL_WIDTHS.left.max, fallback.left), + stage: 1, + right: clampFinite(parsed.right, PANEL_WIDTHS.right.min, PANEL_WIDTHS.right.max, fallback.right), + }; + } catch { + return fallback; + } +} + +export function persistPanelLayout(layout: PanelLayout, storage?: Pick): void { + try { + const target = storage ?? (typeof window !== "undefined" ? window.localStorage : undefined); + if (!target) return; + target.setItem(PANEL_LAYOUT_PREF_KEY, JSON.stringify({ left: layout.left, right: layout.right })); + } catch { + /* localStorage can be unavailable in privacy-restricted contexts. */ + } +} + +export function panelWidthBounds( + target: PanelResizeTarget, + layout: PanelLayout, + viewportWidth: number, + isCompact: boolean, + isMid: boolean, +): { min: number; max: number } { + if (target === "left") { + const floorCap = viewportWidth - layout.right - STAGE_WIDTH_FLOOR - WORKSPACE_EDGE_ALLOWANCE; + const computedMax = floorCap >= PANEL_WIDTHS.left.min ? Math.min(PANEL_WIDTHS.left.max, floorCap) : PANEL_WIDTHS.left.max; + return { + min: PANEL_WIDTHS.left.min, + max: Math.max(layout.left, computedMax), + }; + } + const leftInFlow = isCompact || isMid ? 0 : layout.left; + const floorCap = viewportWidth - leftInFlow - STAGE_WIDTH_FLOOR - WORKSPACE_EDGE_ALLOWANCE; + const computedMax = floorCap >= PANEL_WIDTHS.right.min ? Math.min(PANEL_WIDTHS.right.max, floorCap) : PANEL_WIDTHS.right.max; + return { + min: PANEL_WIDTHS.right.min, + max: Math.max(layout.right, computedMax), + }; +} + +export function setPanelWidth(layout: PanelLayout, target: PanelResizeTarget, width: number, bounds: { min: number; max: number }): PanelLayout { + return { ...layout, [target]: clamp(width, bounds.min, bounds.max) }; +} + function initials(name: string): string { return name.replace(/[^A-Za-z· ]/g, "").split(/[ ·]/).filter(Boolean).map((s) => s[0]).slice(0, 2).join("").toUpperCase() || "?"; } @@ -152,7 +218,8 @@ export function RoomShell({ roomId, me, onLeave, onSignOut, proof }: { roomId: s // Default the side panels lean (binder + Copilot) so the work surface gets the width budget -- // the contract makes it the focus, and an idle Copilot does not need 380px. Both stay inside the // resize clamps (left 176-380, right 280-560), so the user can widen either by dragging. - const [layout, setLayout] = useState({ left: 232, stage: 1, right: 340 }); + const [layout, setLayout] = useState(() => readPersistedPanelLayout()); + const layoutRef = useRef(layout); const [copilotTab, setCopilotTab] = useState<"public" | "private">("public"); const arts = store.listArtifacts(roomId); // Notebook-first: every room lands on the note surface — bankers start by jotting. @@ -515,35 +582,40 @@ export function RoomShell({ roomId, me, onLeave, onSignOut, proof }: { roomId: s : []), ]; const startResize = (target: "left" | "right", startX: number) => { - const start = layout; + const start = layoutRef.current; // Stage floor: cap panel drag so the center Work Surface can't be squeezed below ~760px on desktop. // When the floor is unachievable at the current width (narrow desktops), fall back to the normal // max instead of forcing horizontal overflow. The binder counts as 0 when it floats (<=1199px). - const STAGE_FLOOR = 760, EDGES = 30; const move = (ev: PointerEvent) => { const dx = ev.clientX - startX; - setLayout((cur) => { - const vw = typeof window !== "undefined" ? window.innerWidth : 1440; - if (target === "left") { - const floorCap = vw - cur.right - STAGE_FLOOR - EDGES; - const cap = floorCap >= 176 ? Math.min(380, floorCap) : 380; - return { ...cur, left: clamp(start.left + dx, 176, cap) }; - } - const leftInFlow = isCompact || isMid ? 0 : cur.left; - const floorCap = vw - leftInFlow - STAGE_FLOOR - EDGES; - const cap = floorCap >= 280 ? Math.min(560, floorCap) : 560; - return { ...cur, right: clamp(start.right - dx, 280, cap) }; - }); + const current = layoutRef.current; + const vw = typeof window !== "undefined" ? window.innerWidth : 1440; + const desiredWidth = target === "left" ? start.left + dx : start.right - dx; + const next = setPanelWidth(current, target, desiredWidth, panelWidthBounds(target, current, vw, isCompact, isMid)); + layoutRef.current = next; + setLayout(next); }; const up = () => { window.removeEventListener("pointermove", move); window.removeEventListener("pointerup", up); document.body.classList.remove("r-resizing"); + persistPanelLayout(layoutRef.current); }; document.body.classList.add("r-resizing"); window.addEventListener("pointermove", move); window.addEventListener("pointerup", up); }; + const resizePanelTo = (target: PanelResizeTarget, width: number) => { + const current = layoutRef.current; + const viewportWidth = typeof window !== "undefined" ? window.innerWidth : 1440; + const next = setPanelWidth(current, target, width, panelWidthBounds(target, current, viewportWidth, isCompact, isMid)); + layoutRef.current = next; + setLayout(next); + persistPanelLayout(next); + }; + const viewportWidth = typeof window !== "undefined" ? window.innerWidth : 1440; + const leftPanelBounds = panelWidthBounds("left", layout, viewportWidth, isCompact, isMid); + const rightPanelBounds = panelWidthBounds("right", layout, viewportWidth, isCompact, isMid); // One definition of the panel toggles, placed either in the top bar (compact/mid) or the // settings panel (wide) — never both at once, so it never renders twice. @@ -636,17 +708,17 @@ export function RoomShell({ roomId, me, onLeave, onSignOut, proof }: { roomId: s
- {room.starterBackfill === "pending" ? "Sample still loading." : "Sample workspace."}{" "} - Companies, sources, messages, and traces are synthetic{room.starterBackfill === "pending" ? "; missing sample artifacts retry automatically" : ""}. + {room.starterBackfill === "pending" ? "Live runtime · sample data loading." : "Live runtime · sample data."}{" "} + Convex, collaboration, and NodeAgent are live; seeded companies, sources, messages, and traces are synthetic{room.starterBackfill === "pending" ? "; missing sample artifacts retry automatically" : ""}.
) : null}
{show.left && } - {show.left && startResize("left", x)} />} + {show.left && resizePanelTo("left", width)} onPointerDown={(x) => startResize("left", x)} />} {(!isCompact || show.stage) && } - {show.copilot && startResize("right", x)} />} + {show.copilot && resizePanelTo("right", width)} onPointerDown={(x) => startResize("right", x)} />} {show.copilot && ( void }) { +export function ResizeHandle({ + label, + value, + min, + max, + onResize, + onPointerDown, +}: { + label: string; + value: number; + min: number; + max: number; + onResize: (width: number) => void; + onPointerDown: (clientX: number) => void; +}) { return ( + ))}
)} {surfaceKey !== "secondary" && ( - )} @@ -411,7 +437,7 @@ function ArtifactSurface({ roomId, me, proof, artId, onArt, style, surfaceKey = ) : workArtifactsOpen ? ( }> - { onArt(id); setWorkArtifactsOpen(false); }} /> + { onArt(id); setWorkArtifactsOpen(false); }} /> ) : graphOpen ? ( }> @@ -2192,6 +2218,7 @@ export function GenericSheet({ roomId, me, art, proof, onError }: { roomId: stri const span = mergeAnchor.get(id); const raw = art.elements[id]?.value; const payload = asCellPayload(raw); + const fontColorCss = spreadsheetCellFontColorCss(art, id); const value = displayCellValue(raw); const locked = !!lockedByOther(store, art.id, id, me); const proposed = !!proposalFor(proposals, art.id, id) || draftedFor(store, roomId, art.id, id); @@ -2204,7 +2231,7 @@ export function GenericSheet({ roomId, me, art, proof, onError }: { roomId: stri const metaTitle = evidenceTitle(payload); const cls = "r-cell" + (isNumberLikeCell(raw) ? " num" : "") + (locked ? " locked" : "") + (proposed ? " proposed" : "") + (committed ? " committed rm-wet" : "") + (hasVisibleEvidence ? " evidence" : "") + (showFormulaMarker ? " formula" : "") + (sel === id ? " sel fx-sel" : ""); return ( - { setSel(id); (e.currentTarget.closest("table") as HTMLElement | null)?.focus(); }} onDoubleClick={() => { setEditingId(id); setEditDraft(value); }}> + { setSel(id); (e.currentTarget.closest("table") as HTMLElement | null)?.focus(); }} onDoubleClick={() => { setEditingId(id); setEditDraft(value); }}> {editingId === id ? (