From ea797751f412023e0cef41a780726e8403837caa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tuncay=20=C3=96lmez?= Date: Wed, 22 Jul 2026 09:00:00 +0300 Subject: [PATCH 01/40] Enforce cross-mode worktree ownership --- src/integration/binding.js | 40 +++++++++++++++++++++++++++++++ src/run/ownership.js | 39 +++++++++++++++++++++++++++++++ src/supervise/execution.js | 48 ++++++++++++++++++++------------------ 3 files changed, 104 insertions(+), 23 deletions(-) diff --git a/src/integration/binding.js b/src/integration/binding.js index a9027b4..b78b03e 100644 --- a/src/integration/binding.js +++ b/src/integration/binding.js @@ -7,6 +7,12 @@ const { selectManagedBackend, validateCodexCapabilitySnapshot, } = require("./capabilities"); +const { + OWNERSHIP_SCHEMA_VERSION, + findOwnershipConflict, + loadOwnershipRecords, + validateOwnershipRecord, +} = require("../run/ownership"); const { writeJsonAtomic } = require("../workflow/state"); const HOST_BINDING_SCHEMA_VERSION = "host-binding/v1"; @@ -255,6 +261,39 @@ function bindingPath(found) { return path.join(found.runRoot, "integration", "host-binding.json"); } +function bindingOwnershipPath(found) { + return path.join(found.runRoot, "integration", "ownership.json"); +} + +function claimBindingWorktree(found, binding) { + if (!binding.references.worktree) return null; + if (!binding.workflow.taskId || !binding.workflow.checkpointId) { + throw new Error("Host binding worktree ownership requires a task and active checkpoint identity."); + } + const filePath = bindingOwnershipPath(found); + const requested = validateOwnershipRecord({ + schemaVersion: OWNERSHIP_SCHEMA_VERSION, + runId: binding.workflow.runId, + taskId: binding.workflow.taskId, + checkpointId: binding.workflow.checkpointId, + owner: binding.execution.owner, + backend: binding.execution.backend, + status: "active", + createdAt: binding.provenance.recordedAt, + cleanupAuthority: binding.execution.owner === "managed" ? "cewp-core" : "host-owner", + worktree: binding.references.worktree, + }); + const existing = loadOwnershipRecords(found.repoRoot, { excludePath: filePath }); + const conflict = findOwnershipConflict(existing, requested, { repoRoot: found.repoRoot }); + if (conflict) { + throw new Error( + `Host binding execution ownership conflict with ${conflict.owner} run ${conflict.runId} task ${conflict.taskId}.`, + ); + } + writeJsonAtomic(filePath, requested); + return requested; +} + function createHostBinding(found, candidate, options = {}) { const binding = validateHostBinding(candidate, found); if (!options.capabilities) throw new Error("Host binding requires a versioned capability snapshot."); @@ -264,6 +303,7 @@ function createHostBinding(found, candidate, options = {}) { throw new Error(`Host binding already exists for workflow run ${found.run.runId}.`); } fs.mkdirSync(path.dirname(filePath), { recursive: true }); + claimBindingWorktree(found, binding); writeJsonAtomic(filePath, binding); return binding; } diff --git a/src/run/ownership.js b/src/run/ownership.js index ebf7846..9bd88a2 100644 --- a/src/run/ownership.js +++ b/src/run/ownership.js @@ -1,5 +1,6 @@ "use strict"; +const fs = require("node:fs"); const path = require("node:path"); const { normalizeComparePath } = require("../lib/paths"); @@ -76,11 +77,49 @@ function findOwnershipConflict(records, requested, options = {}) { return undefined; } +function loadOwnershipRecords(repoRoot, options = {}) { + const recordsRoot = path.join(repoRoot, ".cewp"); + if (!fs.existsSync(recordsRoot)) return []; + const excludedPath = options.excludePath + ? normalizeComparePath(path.resolve(options.excludePath)) + : null; + const records = []; + const pending = [recordsRoot]; + + while (pending.length > 0) { + const directory = pending.pop(); + for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { + const entryPath = path.join(directory, entry.name); + if (entry.isDirectory() && !entry.isSymbolicLink()) { + pending.push(entryPath); + continue; + } + if ( + !entry.isFile() + || entry.name !== "ownership.json" + || normalizeComparePath(path.resolve(entryPath)) === excludedPath + ) { + continue; + } + let record; + try { + record = JSON.parse(fs.readFileSync(entryPath, "utf8")); + } catch (error) { + throw new Error(`Invalid execution ownership registry entry ${entryPath}: ${error.message}`); + } + records.push(validateOwnershipRecord(record)); + } + } + + return records; +} + module.exports = { EXECUTION_OWNERS, MANAGED_BACKENDS, OWNERSHIP_SCHEMA_VERSION, findOwnershipConflict, + loadOwnershipRecords, normalizeWorktreePath, validateOwnershipRecord, }; diff --git a/src/supervise/execution.js b/src/supervise/execution.js index 3381da7..22a33ad 100644 --- a/src/supervise/execution.js +++ b/src/supervise/execution.js @@ -23,6 +23,7 @@ const { } = require("../run/adapters/codex-exec"); const { OWNERSHIP_SCHEMA_VERSION, + loadOwnershipRecords, validateOwnershipRecord, } = require("../run/ownership"); const { @@ -210,27 +211,7 @@ function createOwnedWorktree(found, startedAt) { if (continued) return continued; const paths = getWorktreePaths(found); - if (fs.existsSync(paths.worktreePath)) { - throw new Error(`Managed checkpoint worktree already exists: ${paths.worktreePath}`); - } - const branchProbe = getGitOutput(["show-ref", "--verify", "--quiet", `refs/heads/${paths.branch}`], found.repoRoot); - if (branchProbe.status === 0) { - throw new Error(`Managed checkpoint branch already exists: ${paths.branch}`); - } - - fs.mkdirSync(path.dirname(paths.worktreePath), { recursive: true }); - const created = getGitOutput([ - "worktree", - "add", - paths.worktreePath, - "-b", - paths.branch, - checkpointBaseCommit(found.run, found.run.tasks[0]), - ], found.repoRoot); - if (created.status !== 0) { - throw new Error(`Failed to create managed checkpoint worktree: ${(created.stderr || created.stdout || "").trim()}`); - } - + const ownershipPath = path.join(found.runRoot, "ownership.json"); const ownership = validateOwnershipRecord({ schemaVersion: OWNERSHIP_SCHEMA_VERSION, runId: found.runId, @@ -254,13 +235,34 @@ function createOwnedWorktree(found, startedAt) { app: false, notification: false, }, - ownershipRecords: [], + ownershipRecords: loadOwnershipRecords(found.repoRoot, { excludePath: ownershipPath }), requestedOwnership: ownership, }, { repoRoot: found.repoRoot }); if (!gate.allowed) { throw new Error(`Controlled operation blocked: ${gate.reason}`); } - writeJsonAtomic(path.join(found.runRoot, "ownership.json"), ownership); + if (fs.existsSync(paths.worktreePath)) { + throw new Error(`Managed checkpoint worktree already exists: ${paths.worktreePath}`); + } + const branchProbe = getGitOutput(["show-ref", "--verify", "--quiet", `refs/heads/${paths.branch}`], found.repoRoot); + if (branchProbe.status === 0) { + throw new Error(`Managed checkpoint branch already exists: ${paths.branch}`); + } + + fs.mkdirSync(path.dirname(paths.worktreePath), { recursive: true }); + const created = getGitOutput([ + "worktree", + "add", + paths.worktreePath, + "-b", + paths.branch, + checkpointBaseCommit(found.run, found.run.tasks[0]), + ], found.repoRoot); + if (created.status !== 0) { + throw new Error(`Failed to create managed checkpoint worktree: ${(created.stderr || created.stdout || "").trim()}`); + } + + writeJsonAtomic(ownershipPath, ownership); return { ...paths, ownership }; } From 79a0a6ed6820c546cb9fa42f01eab28a55dd8200 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tuncay=20=C3=96lmez?= Date: Wed, 22 Jul 2026 09:30:00 +0300 Subject: [PATCH 02/40] Test cross-mode worktree ownership --- tests/contracts/integration-binding.js | 52 ++++++++++++++++++++++++- tests/contracts/supervised-execution.js | 52 +++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 1 deletion(-) diff --git a/tests/contracts/integration-binding.js b/tests/contracts/integration-binding.js index 91da018..0fd7515 100644 --- a/tests/contracts/integration-binding.js +++ b/tests/contracts/integration-binding.js @@ -12,7 +12,7 @@ const { createHostBinding, loadHostBinding, } = require("../../src/integration/binding"); -const { loadWorkflowRun } = require("../../src/workflow/state"); +const { loadWorkflowRun, startWorkflowTask } = require("../../src/workflow/state"); function assertThrows(action, expected, label) { let error; @@ -129,6 +129,56 @@ function main() { cleanupRepo(managedRepo); } + const conflictRepo = makeTempRepo("cewp-integration-ownership-conflict-"); + try { + const native = approveWorkflow(conflictRepo, nativeDefinition()); + let nativeFound = loadWorkflowRun(conflictRepo, native.runId); + const started = startWorkflowTask(nativeFound, "implement-example", { + now: new Date("2026-07-18T12:01:00.000Z"), + }); + nativeFound = loadWorkflowRun(conflictRepo, native.runId); + const sharedWorktree = path.join(conflictRepo, "..", ".cewp-worktrees", "shared-task"); + const managedOwnershipPath = path.join( + conflictRepo, + ".cewp", + "supervised-runs", + "managed-conflict", + "ownership.json", + ); + fs.mkdirSync(path.dirname(managedOwnershipPath), { recursive: true }); + fs.writeFileSync(managedOwnershipPath, `${JSON.stringify({ + schemaVersion: "execution-ownership/v1", + runId: "managed-conflict", + taskId: "implement-example", + checkpointId: "implement-example", + owner: "managed", + backend: "codex-exec", + status: "active", + createdAt: "2026-07-18T12:00:00.000Z", + cleanupAuthority: "cewp-core", + worktree: { id: "shared-task", path: sharedWorktree }, + }, null, 2)}\n`); + + const conflictingBinding = explicitBinding(native.runId); + conflictingBinding.workflow = { + runId: native.runId, + taskId: "implement-example", + checkpointId: started.checkpoint.checkpointId, + }; + conflictingBinding.references.worktree = { id: "shared-task", path: sharedWorktree }; + assertThrows( + () => createHostBinding(nativeFound, conflictingBinding, { capabilities: supportedSnapshot() }), + /execution ownership conflict/, + "native host binding cannot claim an active managed task worktree", + ); + assert( + !fs.existsSync(path.join(nativeFound.runRoot, "integration", "host-binding.json")), + "conflicting native binding is not persisted", + ); + } finally { + cleanupRepo(conflictRepo); + } + const coreRun = JSON.parse(fs.readFileSync(found.runPath, "utf8")); assert(coreRun.host === undefined && coreRun.references === undefined, "provider ids stay outside core schema"); diff --git a/tests/contracts/supervised-execution.js b/tests/contracts/supervised-execution.js index cc8ad5b..bd7d198 100644 --- a/tests/contracts/supervised-execution.js +++ b/tests/contracts/supervised-execution.js @@ -48,6 +48,57 @@ function createApprovedRun(repoRoot) { return runId; } +function runNativeOwnershipConflictContract() { + const repoRoot = makeTempRepo("cewp-supervised-native-conflict-"); + const fake = createFakeCodexAdapter(); + try { + const runId = createApprovedRun(repoRoot); + assert(runNode(cewpCli, ["policy", "set", "full-authority"], repoRoot).status === 0, "fixture grants worker authority"); + const runPath = path.join(repoRoot, ".cewp", "supervised-runs", runId, "run.json"); + const run = JSON.parse(fs.readFileSync(runPath, "utf8")); + const taskId = run.tasks[0].id; + const targetWorktree = path.resolve( + repoRoot, + "..", + ".cewp-worktrees", + path.basename(repoRoot), + runId, + taskId, + ); + const nativeOwnershipPath = path.join( + repoRoot, + ".cewp", + "workflow-runs", + "native-owner", + "integration", + "ownership.json", + ); + fs.mkdirSync(path.dirname(nativeOwnershipPath), { recursive: true }); + fs.writeFileSync(nativeOwnershipPath, `${JSON.stringify({ + schemaVersion: "execution-ownership/v1", + runId: "native-owner", + taskId, + checkpointId: `${taskId}-attempt-0001`, + owner: "native", + backend: null, + status: "active", + createdAt: "2026-07-18T12:00:00.000Z", + cleanupAuthority: "host-owner", + worktree: { id: `${runId}:${taskId}`, path: targetWorktree }, + }, null, 2)}\n`); + + const result = runNode(cewpCli, [ + "supervise", "execute", runId, "--yes", "--timeout", "20", "--json", + ], repoRoot, { env: fake.env }); + assert(result.status === 1, "managed dispatch rejects a native-owned task worktree"); + assert(result.stderr.includes("execution-ownership-conflict"), "ownership refusal is actionable"); + assert(!fs.existsSync(targetWorktree), "conflicting managed worktree is not created"); + } finally { + fs.rmSync(fake.fakeRoot, { recursive: true, force: true }); + cleanupRepo(repoRoot); + } +} + function runSupervisedExecutionContract() { const repoRoot = makeTempRepo("cewp-supervised-exec-"); const fake = createFakeCodexAdapter(); @@ -209,6 +260,7 @@ function runSupervisedExecutionContract() { } try { + runNativeOwnershipConflictContract(); runSupervisedExecutionContract(); console.log("[PASS] supervised dispatch preserves ownership, scope, and usage truth"); } catch (error) { From 1dafc4f15a6688daad01cc78e3f8d589d671d2d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tuncay=20=C3=96lmez?= Date: Wed, 22 Jul 2026 10:00:00 +0300 Subject: [PATCH 03/40] Add operator-approved Codex effort policy --- src/cli/parse.js | 38 ++++- src/cli/usage.js | 1 + src/integration/effort-policy.js | 265 +++++++++++++++++++++++++++++++ src/run/adapters/codex-exec.js | 10 +- src/supervise/cli.js | 14 ++ src/supervise/execution.js | 14 ++ src/supervise/review.js | 9 ++ 7 files changed, 349 insertions(+), 2 deletions(-) create mode 100644 src/integration/effort-policy.js diff --git a/src/cli/parse.js b/src/cli/parse.js index 15b3c04..43cd61c 100644 --- a/src/cli/parse.js +++ b/src/cli/parse.js @@ -74,6 +74,10 @@ function parseArgs(argv) { workerId: undefined, templateName: undefined, compilerDigest: undefined, + operation: undefined, + taskClass: undefined, + model: undefined, + effort: undefined, }; if (argv[0] === "--help" || argv[0] === "-h") { @@ -203,7 +207,7 @@ function parseArgs(argv) { if ( args.command === "supervise" && [ - "approve", "status", "execute", "verify", "retry", "review", "receipt", "finalize", + "approve", "status", "execute", "verify", "retry", "review", "receipt", "finalize", "effort", "revise", "pause", "resume", "add-budget", "rollback", "cancel", "abandon", "block", "continue", "reassign", ].includes(args.subcommand) && index === 2 @@ -213,6 +217,38 @@ function parseArgs(argv) { continue; } + if (args.command === "supervise" && arg === "--operation") { + const value = argv[index + 1]; + if (!value || value.startsWith("--")) throw new Error("--operation requires an effort-policy operation."); + args.operation = value; + index += 1; + continue; + } + + if (args.command === "supervise" && arg === "--task-class") { + const value = argv[index + 1]; + if (!value || value.startsWith("--")) throw new Error("--task-class requires a Codex task class."); + args.taskClass = value; + index += 1; + continue; + } + + if (args.command === "supervise" && arg === "--model") { + const value = argv[index + 1]; + if (!value || value.startsWith("--")) throw new Error("--model requires an explicit Codex model."); + args.model = value; + index += 1; + continue; + } + + if (args.command === "supervise" && arg === "--effort") { + const value = argv[index + 1]; + if (!value || value.startsWith("--")) throw new Error("--effort requires a supported Codex reasoning effort."); + args.effort = value; + index += 1; + continue; + } + if ( ( args.command === "supervise" diff --git a/src/cli/usage.js b/src/cli/usage.js index b804c0c..d63d86b 100644 --- a/src/cli/usage.js +++ b/src/cli/usage.js @@ -31,6 +31,7 @@ Usage: cewp supervise plan --proposal [--from ] [--source-kind ] [--json] cewp supervise approve [run-id] [--allow-test-authoring] --yes [--json] cewp supervise status [run-id] [--json] + cewp supervise effort [run-id] --operation --task-class [--model ] [--effort ] --yes [--json] cewp supervise execute [run-id] --yes [--timeout ] [--json] cewp supervise verify [run-id] [--timeout ] [--json] cewp supervise retry [run-id] --yes [--timeout ] [--json] diff --git a/src/integration/effort-policy.js b/src/integration/effort-policy.js new file mode 100644 index 0000000..27b98d4 --- /dev/null +++ b/src/integration/effort-policy.js @@ -0,0 +1,265 @@ +"use strict"; + +const fs = require("node:fs"); +const crypto = require("node:crypto"); +const path = require("node:path"); +const { appendEvent, findSupervisedRun, getNextAction } = require("../supervise/state"); +const { writeJsonAtomic } = require("../workflow/state"); + +const CODEX_EFFORT_POLICY_SCHEMA_VERSION = "codex-effort-policy/v1"; +const CODEX_TASK_CLASSES = Object.freeze([ + "fast-exploration", + "demanding-implementation", + "high-effort-independent-review", +]); +const CODEX_EFFORT_OPERATIONS = Object.freeze(["implementation", "repair", "reviewer"]); +const CODEX_REASONING_EFFORTS = Object.freeze(["minimal", "low", "medium", "high", "xhigh"]); + +function requiredChoice(value, allowed, label) { + if (!allowed.includes(value)) { + throw new Error(`${label} must be one of: ${allowed.join(", ")}.`); + } + return value; +} + +function optionalSelection(value, label) { + if (value === undefined || value === null) return { status: "unknown", value: null }; + if (typeof value !== "string" || value.trim().length === 0 || value.length > 128 || /[\u0000-\u001f]/.test(value)) { + throw new Error(`${label} must be bounded non-empty text.`); + } + return { status: "explicit", value: value.trim() }; +} + +function effortPolicyPath(found) { + return path.join(found.runRoot, "integration", "codex-effort-policy.json"); +} + +function selectionDigest(operation, assignment) { + const content = JSON.stringify({ + operation, + workflow: assignment.workflow, + taskClass: assignment.taskClass, + model: assignment.requested.model, + effort: assignment.requested.effort, + }); + return `sha256:${crypto.createHash("sha256").update(content).digest("hex")}`; +} + +function assignmentSnapshot(assignment) { + if (!assignment) return null; + return { + workflow: assignment.workflow, + taskClass: assignment.taskClass, + requested: assignment.requested, + approval: assignment.approval, + }; +} + +function validateAssignment(operation, assignment, found, options = {}) { + if (!assignment || typeof assignment !== "object") { + throw new Error(`Invalid Codex effort assignment for ${operation}.`); + } + requiredChoice(assignment.taskClass, CODEX_TASK_CLASSES, `${operation}.taskClass`); + if ( + !assignment.workflow + || !Number.isInteger(assignment.workflow.planRevision) + || assignment.workflow.planRevision <= 0 + || typeof assignment.workflow.checkpointId !== "string" + || assignment.workflow.checkpointId.length === 0 + ) { + throw new Error(`Invalid Codex effort workflow binding for ${operation}.`); + } + const model = optionalSelection( + assignment.requested && assignment.requested.model && assignment.requested.model.value, + `${operation}.requested.model`, + ); + const effortValue = assignment.requested && assignment.requested.effort && assignment.requested.effort.value; + if (effortValue !== null && effortValue !== undefined) { + requiredChoice(effortValue, CODEX_REASONING_EFFORTS, `${operation}.requested.effort`); + } + const effort = optionalSelection(effortValue, `${operation}.requested.effort`); + const normalized = { + ...assignment, + requested: { model, effort }, + }; + if ( + !assignment.approval + || assignment.approval.kind !== "operator" + || assignment.approval.selectionDigest !== selectionDigest(operation, normalized) + ) { + throw new Error(`Codex effort assignment for ${operation} is not operator-approved or was modified.`); + } + if ( + options.allowStale !== true + && ( + assignment.workflow.planRevision !== found.run.planRevision + || assignment.workflow.checkpointId !== found.run.tasks[0].id + ) + ) { + throw new Error(`Codex effort assignment for ${operation} was not approved for the current plan revision and checkpoint.`); + } + return normalized; +} + +function loadCodexEffortPolicy(found, options = {}) { + const filePath = effortPolicyPath(found); + if (!fs.existsSync(filePath)) return null; + const policy = JSON.parse(fs.readFileSync(filePath, "utf8")); + if (policy.schemaVersion !== CODEX_EFFORT_POLICY_SCHEMA_VERSION || policy.runId !== found.runId) { + throw new Error(`Invalid Codex effort policy for run ${found.runId}.`); + } + if (policy.provider !== "codex" || policy.automaticModelRouting !== false) { + throw new Error("Invalid Codex effort policy provider or routing boundary."); + } + return { + ...policy, + assignments: Object.fromEntries(Object.entries(policy.assignments || {}).map(([operation, assignment]) => { + requiredChoice(operation, CODEX_EFFORT_OPERATIONS, "effort policy operation"); + return [operation, validateAssignment(operation, assignment, found, options)]; + })), + }; +} + +function approveCodexEffortPolicy(options = {}) { + if (!options.yes) { + throw new Error("Codex effort policy changes require explicit operator approval with --yes."); + } + const found = findSupervisedRun(options); + const operation = requiredChoice(options.operation, CODEX_EFFORT_OPERATIONS, "--operation"); + const taskClass = requiredChoice(options.taskClass, CODEX_TASK_CLASSES, "--task-class"); + if (options.effort !== undefined) { + requiredChoice(options.effort, CODEX_REASONING_EFFORTS, "--effort"); + } + const previous = loadCodexEffortPolicy(found, { allowStale: true }); + const revision = previous ? previous.revision + 1 : 1; + const approvedAt = new Date().toISOString(); + const assignment = { + workflow: { + planRevision: found.run.planRevision, + checkpointId: found.run.tasks[0].id, + }, + taskClass, + requested: { + model: optionalSelection(options.model, "--model"), + effort: optionalSelection(options.effort, "--effort"), + }, + approval: { + kind: "operator", + event: "codex-effort-policy-approved", + revision, + approvedAt, + selectionDigest: null, + }, + }; + assignment.approval.selectionDigest = selectionDigest(operation, assignment); + const previousAssignment = previous ? previous.assignments[operation] || null : null; + const policy = { + schemaVersion: CODEX_EFFORT_POLICY_SCHEMA_VERSION, + runId: found.runId, + provider: "codex", + automaticModelRouting: false, + revision, + updatedAt: approvedAt, + assignments: { + ...(previous ? previous.assignments : {}), + [operation]: assignment, + }, + history: [ + ...(previous ? previous.history : []), + { + revision, + operation, + taskClass, + approvedAt, + previousRevision: previous ? previous.revision : null, + previous: assignmentSnapshot(previousAssignment), + next: assignmentSnapshot(assignment), + }, + ], + }; + const filePath = effortPolicyPath(found); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + writeJsonAtomic(filePath, policy); + appendEvent(found.runRoot, { + schemaVersion: "supervised-event/v1-beta", + timestamp: approvedAt, + type: "codex-effort-policy-approved", + runId: found.runId, + planRevision: found.run.planRevision, + checkpointId: found.run.tasks[0].id, + actor: "operator", + operation, + revision, + selectionDigest: assignment.approval.selectionDigest, + }); + return { + run: found.run, + effortPolicy: policy, + nextAction: getNextAction(found.run), + }; +} + +function resolveCodexEffortForDispatch(found, operation) { + requiredChoice(operation, CODEX_EFFORT_OPERATIONS, "Codex dispatch operation"); + const policy = loadCodexEffortPolicy(found); + const assignment = policy && policy.assignments[operation]; + if (!assignment) { + return { + model: undefined, + effort: undefined, + evidence: { + taskClass: null, + policyRevision: policy ? policy.revision : null, + selectedModel: { status: "unknown", value: null }, + selectedEffort: { status: "unknown", value: null }, + effectiveModel: { status: "unknown", value: null, source: "not-explicitly-selected" }, + effectiveEffort: { status: "unknown", value: null, source: "not-explicitly-selected" }, + }, + }; + } + const model = assignment.requested.model.value || undefined; + const effort = assignment.requested.effort.value || undefined; + return { + model, + effort, + evidence: { + taskClass: assignment.taskClass, + policyRevision: policy.revision, + selectedModel: assignment.requested.model, + selectedEffort: assignment.requested.effort, + effectiveModel: { status: "unknown", value: null, source: model ? "awaiting-supported-turn-evidence" : "not-explicitly-selected" }, + effectiveEffort: { status: "unknown", value: null, source: effort ? "awaiting-supported-turn-evidence" : "not-explicitly-selected" }, + }, + }; +} + +function confirmCodexEffortEvidence(evidence, usage) { + const confirmed = usage && usage.label === "observed"; + const effective = (selected) => ( + confirmed && selected && selected.status === "explicit" + ? { status: "known", value: selected.value, source: "codex-exec-turn-completed-usage" } + : { + status: "unknown", + value: null, + source: selected && selected.status === "explicit" + ? "supported-turn-evidence-unavailable" + : "not-explicitly-selected", + } + ); + return { + ...evidence, + effectiveModel: effective(evidence.selectedModel), + effectiveEffort: effective(evidence.selectedEffort), + }; +} + +module.exports = { + CODEX_EFFORT_OPERATIONS, + CODEX_EFFORT_POLICY_SCHEMA_VERSION, + CODEX_REASONING_EFFORTS, + CODEX_TASK_CLASSES, + approveCodexEffortPolicy, + confirmCodexEffortEvidence, + loadCodexEffortPolicy, + resolveCodexEffortForDispatch, +}; diff --git a/src/run/adapters/codex-exec.js b/src/run/adapters/codex-exec.js index 2e91d81..b5d3ec8 100644 --- a/src/run/adapters/codex-exec.js +++ b/src/run/adapters/codex-exec.js @@ -198,14 +198,20 @@ function buildCodexExecInvocation({ outputLastMessagePath, sandbox = "workspace-write", structuredJson = false, + model, + effort, }) { const formatArgs = structuredJson ? ["--json"] : []; + const modelArgs = model ? ["--model", model] : []; + const effortArgs = effort ? ["--config", `model_reasoning_effort="${effort}"`] : []; return { command: command || "codex", args: [ ...prefixArgs, "exec", ...formatArgs, + ...modelArgs, + ...effortArgs, "--cd", worktreePath, "--sandbox", @@ -218,7 +224,7 @@ function buildCodexExecInvocation({ }; } -function runCodexExecAdapter({ worktreePath, promptPath, outputLastMessagePath, timeoutSeconds, sandbox = "workspace-write", structuredJson = false }) { +function runCodexExecAdapter({ worktreePath, promptPath, outputLastMessagePath, timeoutSeconds, sandbox = "workspace-write", structuredJson = false, model, effort }) { validateTimeoutSeconds(timeoutSeconds); const prompt = fs.readFileSync(promptPath, "utf8"); const invocation = buildCodexExecInvocation({ @@ -229,6 +235,8 @@ function runCodexExecAdapter({ worktreePath, promptPath, outputLastMessagePath, outputLastMessagePath, sandbox, structuredJson, + model, + effort, }); return childProcess.spawnSync(invocation.command, invocation.args, { diff --git a/src/supervise/cli.js b/src/supervise/cli.js index cea23e3..f8811f4 100644 --- a/src/supervise/cli.js +++ b/src/supervise/cli.js @@ -10,6 +10,7 @@ const { verifySupervisedCheckpoint } = require("./verification"); const { reviewSupervisedCheckpoint } = require("./review"); const { finalizeSupervisedRun, previewSupervisedReceipt } = require("./receipt"); const { runSupervisedControl } = require("./controls"); +const { approveCodexEffortPolicy } = require("../integration/effort-policy"); const CONTROL_COMMANDS = new Set([ "revise", "pause", "resume", "add-budget", "rollback", "cancel", "abandon", "block", "continue", "reassign", @@ -99,6 +100,19 @@ function runSupervise(options = {}) { return; } + if (options.subcommand === "effort") { + const result = approveCodexEffortPolicy({ + ...options, + repoRoot: process.cwd(), + }); + if (options.json) { + outputJson("supervise.effort", result); + } else { + printStatus("CEWP Codex effort policy approved", result); + } + return; + } + if (options.subcommand === "execute") { const result = executeSupervisedCheckpoint({ ...options, diff --git a/src/supervise/execution.js b/src/supervise/execution.js index 22a33ad..0320ad1 100644 --- a/src/supervise/execution.js +++ b/src/supervise/execution.js @@ -26,6 +26,10 @@ const { loadOwnershipRecords, validateOwnershipRecord, } = require("../run/ownership"); +const { + confirmCodexEffortEvidence, + resolveCodexEffortForDispatch, +} = require("../integration/effort-policy"); const { appendEvent, findSupervisedRun, @@ -303,6 +307,7 @@ function executeSupervisedCheckpoint(options = {}) { assertPolicyAllows(found.repoRoot, "runWorkers"); assertPolicyAllows(found.repoRoot, "runCommands"); enforceOperationBudget(found, "implementation"); + const codexSelection = resolveCodexEffortForDispatch(found, "implementation"); const startedAt = new Date().toISOString(); const owned = createOwnedWorktree(found, startedAt); @@ -332,6 +337,7 @@ function executeSupervisedCheckpoint(options = {}) { scope: { status: "pending", warnings: [] }, testAuthoring: { policy: found.run.assurance.testAuthoring, status: "pending", violations: [] }, usage: { label: "unknown", value: null }, + codex: codexSelection.evidence, }; let startedRun = { ...found.run, @@ -398,6 +404,8 @@ function executeSupervisedCheckpoint(options = {}) { timeoutSeconds: options.timeoutSeconds, sandbox: "workspace-write", structuredJson: true, + model: codexSelection.model, + effort: codexSelection.effort, }); const remainingOutput = Math.max( 0, @@ -441,6 +449,7 @@ function executeSupervisedCheckpoint(options = {}) { }, testAuthoring, usage, + codex: confirmCodexEffortEvidence(attempt.codex, usage), logs: { stdout: path.relative(found.runRoot, stdoutPath).replace(/\\/g, "/"), stderr: path.relative(found.runRoot, stderrPath).replace(/\\/g, "/"), @@ -551,6 +560,7 @@ function retrySupervisedCheckpoint(options = {}) { } assertPolicyAllows(found.repoRoot, "runWorkers"); enforceOperationBudget(found, "repair"); + const codexSelection = resolveCodexEffortForDispatch(found, "repair"); const repairCount = task.attempts.filter((attempt) => attempt.kind === "repair").length; if (repairCount >= found.run.budget.maxRepairsPerCheckpoint.value) { throw new Error("Checkpoint repair limit is exhausted; explicit budget revision is required."); @@ -594,6 +604,7 @@ function retrySupervisedCheckpoint(options = {}) { scope: { status: "pending", warnings: [] }, testAuthoring: { policy: found.run.assurance.testAuthoring, status: "pending", violations: [] }, usage: { label: "unknown", value: null }, + codex: codexSelection.evidence, }; let startedRun = { ...found.run, @@ -640,6 +651,8 @@ function retrySupervisedCheckpoint(options = {}) { timeoutSeconds: options.timeoutSeconds, sandbox: "workspace-write", structuredJson: true, + model: codexSelection.model, + effort: codexSelection.effort, }); const remainingOutput = Math.max( 0, @@ -681,6 +694,7 @@ function retrySupervisedCheckpoint(options = {}) { }, testAuthoring, usage, + codex: confirmCodexEffortEvidence(attempt.codex, usage), logs: { stdout: path.relative(found.runRoot, stdoutPath).replace(/\\/g, "/"), stderr: path.relative(found.runRoot, stderrPath).replace(/\\/g, "/"), diff --git a/src/supervise/review.js b/src/supervise/review.js index d502c7e..40dbedc 100644 --- a/src/supervise/review.js +++ b/src/supervise/review.js @@ -10,6 +10,10 @@ const { runCodexExecAdapter, } = require("../run/adapters/codex-exec"); const { validateOwnershipRecord } = require("../run/ownership"); +const { + confirmCodexEffortEvidence, + resolveCodexEffortForDispatch, +} = require("../integration/effort-policy"); const { applyThresholdObservation, enforceOperationBudget } = require("./budget"); const { mergeManagedUsage, @@ -80,6 +84,7 @@ function reviewSupervisedCheckpoint(options = {}) { } assertPolicyAllows(found.repoRoot, "runReviewer"); enforceOperationBudget(found, "reviewer"); + const codexSelection = resolveCodexEffortForDispatch(found, "reviewer"); const ownership = validateOwnershipRecord( readJsonFile(path.join(found.runRoot, "ownership.json"), "execution ownership"), ); @@ -114,6 +119,7 @@ function reviewSupervisedCheckpoint(options = {}) { independent: true, status: "executing", startedAt, + codex: codexSelection.evidence, }, }; startedRun.budget.consumed.modelOperations += 1; @@ -147,6 +153,8 @@ function reviewSupervisedCheckpoint(options = {}) { timeoutSeconds: options.timeoutSeconds, sandbox: "read-only", structuredJson: true, + model: codexSelection.model, + effort: codexSelection.effort, }); const remainingOutput = Math.max( 0, @@ -188,6 +196,7 @@ function reviewSupervisedCheckpoint(options = {}) { timedOut, reportPath: path.relative(found.runRoot, reportPath).replace(/\\/g, "/"), usage, + codex: confirmCodexEffortEvidence(startedRun.reviewer.codex, usage), reason: decision ? null : "Reviewer output did not contain a supported Decision line.", From 20b8cd204b16272ed20437873ce1bb2d284c2278 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tuncay=20=C3=96lmez?= Date: Wed, 22 Jul 2026 10:30:00 +0300 Subject: [PATCH 04/40] Test supervised Codex effort controls --- tests/contracts/integration-effort-policy.js | 271 +++++++++++++++++++ tests/harness/lib/fake-adapter.js | 11 + 2 files changed, 282 insertions(+) create mode 100644 tests/contracts/integration-effort-policy.js diff --git a/tests/contracts/integration-effort-policy.js b/tests/contracts/integration-effort-policy.js new file mode 100644 index 0000000..b07f566 --- /dev/null +++ b/tests/contracts/integration-effort-policy.js @@ -0,0 +1,271 @@ +"use strict"; + +const fs = require("node:fs"); +const path = require("node:path"); +const { assert } = require("../harness/lib/assertions"); +const { createFakeCodexAdapter } = require("../harness/lib/fake-adapter"); +const { cleanupRepo, makeTempRepo, runNode } = require("../harness/lib/temp-repo"); + +const cewpCli = path.join(__dirname, "..", "..", "bin", "cewp.js"); + +function parseJson(result, label) { + assert(result.status === 0, `${label} failed: ${result.stderr}`); + return JSON.parse(result.stdout); +} + +function createApprovedRun(repoRoot) { + const planned = parseJson(runNode(cewpCli, [ + "supervise", "plan", + "--goal", "Inspect one bounded behavior", + "--scope", "README.md", + "--verify", "git diff --check", + "--stop", "The bounded behavior is inspected", + "--json", + ], repoRoot), "supervise plan"); + const runId = planned.data.run.runId; + parseJson(runNode(cewpCli, [ + "supervise", "approve", runId, "--yes", "--json", + ], repoRoot), "supervise approve"); + return runId; +} + +function createApprovedRepairRun(repoRoot) { + const verification = "node -e \"const fs=require('fs'); process.exit(fs.readFileSync('README.md','utf8').includes('Fake Codex')?1:0)\""; + const planned = parseJson(runNode(cewpCli, [ + "supervise", "plan", + "--goal", "Repair one bounded regression", + "--scope", "README.md", + "--verify", verification, + "--stop", "The bounded regression is repaired", + "--json", + ], repoRoot), "repair supervise plan"); + const runId = planned.data.run.runId; + parseJson(runNode(cewpCli, ["supervise", "approve", runId, "--yes", "--json"], repoRoot), "repair supervise approve"); + return runId; +} + +function runRepairEffortContract() { + const repoRoot = makeTempRepo("cewp-repair-effort-policy-"); + const fake = createFakeCodexAdapter(); + try { + const runId = createApprovedRepairRun(repoRoot); + assert(runNode(cewpCli, ["policy", "set", "full-authority"], repoRoot).status === 0, "repair fixture grants dispatch authority"); + parseJson(runNode(cewpCli, [ + "supervise", "execute", runId, "--yes", "--timeout", "20", "--json", + ], repoRoot, { env: fake.env }), "execute repair fixture"); + const failedVerification = runNode(cewpCli, [ + "supervise", "verify", runId, "--timeout", "20", "--json", + ], repoRoot); + assert(failedVerification.status === 1, "repair fixture reaches a failed verification gate"); + const failed = JSON.parse(failedVerification.stdout); + assert(failed.data.run.status === "needs-repair", "failed verification exposes bounded repair"); + + parseJson(runNode(cewpCli, [ + "supervise", "effort", runId, + "--operation", "repair", + "--task-class", "demanding-implementation", + "--model", "gpt-test-repair", + "--effort", "medium", + "--yes", "--json", + ], repoRoot), "approve repair effort policy"); + const retried = parseJson(runNode(cewpCli, [ + "supervise", "retry", runId, "--yes", "--timeout", "20", "--json", + ], repoRoot, { + env: { + ...fake.env, + CEWP_FAKE_CODEX_EXPECT_MODEL: "gpt-test-repair", + CEWP_FAKE_CODEX_EXPECT_EFFORT: "medium", + }, + }), "retry with approved effort policy"); + const repairAttempt = retried.data.run.tasks[0].attempts.at(-1); + assert(repairAttempt.kind === "repair", "repair attempt remains explicitly classified"); + assert(repairAttempt.codex.taskClass === "demanding-implementation", "repair evidence retains its task class"); + assert(repairAttempt.codex.effectiveModel.value === "gpt-test-repair", "repair evidence records the effective model"); + assert(repairAttempt.codex.effectiveEffort.value === "medium", "repair evidence records the effective effort"); + } finally { + fs.rmSync(fake.fakeRoot, { recursive: true, force: true }); + cleanupRepo(repoRoot); + } +} + +function runEffortTamperContract() { + const repoRoot = makeTempRepo("cewp-effort-policy-tamper-"); + const fake = createFakeCodexAdapter(); + try { + const runId = createApprovedRun(repoRoot); + parseJson(runNode(cewpCli, [ + "supervise", "effort", runId, + "--operation", "implementation", + "--task-class", "demanding-implementation", + "--model", "gpt-approved", + "--effort", "high", + "--yes", "--json", + ], repoRoot), "approve tamper fixture policy"); + const policyPath = path.join(repoRoot, ".cewp", "supervised-runs", runId, "integration", "codex-effort-policy.json"); + const policy = JSON.parse(fs.readFileSync(policyPath, "utf8")); + policy.assignments.implementation.requested.model.value = "gpt-unapproved-edit"; + fs.writeFileSync(policyPath, `${JSON.stringify(policy, null, 2)}\n`); + assert(runNode(cewpCli, ["policy", "set", "full-authority"], repoRoot).status === 0, "tamper fixture grants dispatch authority"); + const refused = runNode(cewpCli, [ + "supervise", "execute", runId, "--yes", "--timeout", "20", "--json", + ], repoRoot, { env: fake.env }); + assert(refused.status === 1, "modified effort sidecar cannot dispatch"); + assert(refused.stderr.includes("not operator-approved or was modified"), "tamper refusal explains the approval failure"); + } finally { + fs.rmSync(fake.fakeRoot, { recursive: true, force: true }); + cleanupRepo(repoRoot); + } +} + +function runStalePlanRevisionContract() { + const repoRoot = makeTempRepo("cewp-effort-policy-stale-revision-"); + const fake = createFakeCodexAdapter(); + try { + const runId = createApprovedRun(repoRoot); + parseJson(runNode(cewpCli, [ + "supervise", "effort", runId, + "--operation", "implementation", + "--task-class", "demanding-implementation", + "--model", "gpt-old-plan", + "--effort", "high", + "--yes", "--json", + ], repoRoot), "approve old-plan effort policy"); + const revised = parseJson(runNode(cewpCli, [ + "supervise", "revise", runId, + "--goal", "Inspect a revised bounded behavior", + "--json", + ], repoRoot), "revise effort-policy plan"); + assert(revised.data.run.planRevision === 2, "fixture creates a new plan revision"); + parseJson(runNode(cewpCli, ["supervise", "approve", runId, "--yes", "--json"], repoRoot), "approve revised plan"); + assert(runNode(cewpCli, ["policy", "set", "full-authority"], repoRoot).status === 0, "stale revision fixture grants dispatch authority"); + const refused = runNode(cewpCli, [ + "supervise", "execute", runId, "--yes", "--timeout", "20", "--json", + ], repoRoot, { env: fake.env }); + assert(refused.status === 1, "old-plan effort approval cannot dispatch a revised plan"); + assert(refused.stderr.includes("current plan revision"), "stale approval refusal names the plan-revision mismatch"); + } finally { + fs.rmSync(fake.fakeRoot, { recursive: true, force: true }); + cleanupRepo(repoRoot); + } +} + +function main() { + const repoRoot = makeTempRepo("cewp-effort-policy-"); + const fake = createFakeCodexAdapter(); + try { + const runId = createApprovedRun(repoRoot); + const unapproved = runNode(cewpCli, [ + "supervise", "effort", runId, + "--operation", "implementation", + "--task-class", "fast-exploration", + "--json", + ], repoRoot); + assert(unapproved.status === 1, "effort changes require explicit --yes approval"); + assert(unapproved.stderr.includes("explicit operator approval"), "approval refusal is actionable"); + const configured = parseJson(runNode(cewpCli, [ + "supervise", "effort", runId, + "--operation", "implementation", + "--task-class", "fast-exploration", + "--yes", "--json", + ], repoRoot), "supervise effort"); + + assert(configured.command === "supervise.effort", "effort command identifies its public operation"); + const policy = configured.data.effortPolicy; + assert(policy.schemaVersion === "codex-effort-policy/v1", "effort policy is versioned"); + assert(policy.provider === "codex", "provider identity stays in the integration sidecar"); + assert(policy.automaticModelRouting === false, "automatic model routing remains disabled"); + assert(policy.assignments.implementation.taskClass === "fast-exploration", "explicit task class is retained"); + assert(policy.assignments.implementation.requested.model.status === "unknown", "task class does not infer a model"); + assert(policy.assignments.implementation.requested.effort.status === "unknown", "task class does not infer effort"); + assert(policy.assignments.implementation.approval.kind === "operator", "operator approval is recorded"); + const canonicalRun = JSON.parse(fs.readFileSync( + path.join(repoRoot, ".cewp", "supervised-runs", runId, "run.json"), + "utf8", + )); + assert(canonicalRun.effortPolicy === undefined, "provider-specific effort policy stays outside canonical run state"); + + const explicit = parseJson(runNode(cewpCli, [ + "supervise", "effort", runId, + "--operation", "implementation", + "--task-class", "demanding-implementation", + "--model", "gpt-test-explicit", + "--effort", "high", + "--yes", "--json", + ], repoRoot), "approve explicit model and effort"); + assert(explicit.data.effortPolicy.revision === 2, "approved setting change creates a new revision"); + const change = explicit.data.effortPolicy.history.at(-1); + assert(change.previous.taskClass === "fast-exploration", "change history retains the previous task class"); + assert(change.previous.requested.model.status === "unknown", "change history retains the previous unknown model"); + assert(change.next.taskClass === "demanding-implementation", "change history retains the next task class"); + assert(change.next.requested.model.value === "gpt-test-explicit", "change history retains the approved next model"); + assert(change.next.requested.effort.value === "high", "change history retains the approved next effort"); + const approvalEvents = fs.readFileSync( + path.join(repoRoot, ".cewp", "supervised-runs", runId, "events.jsonl"), + "utf8", + ).trim().split("\n").map((line) => JSON.parse(line)) + .filter((event) => event.type === "codex-effort-policy-approved"); + assert(approvalEvents.length === 2, "each effort policy approval is retained in the run event log"); + assert(approvalEvents.at(-1).revision === 2, "the event log identifies the approved policy revision"); + assert( + approvalEvents.at(-1).selectionDigest === explicit.data.effortPolicy.assignments.implementation.approval.selectionDigest, + "the event log binds the operator approval to the selected policy digest", + ); + assert(runNode(cewpCli, ["policy", "set", "full-authority"], repoRoot).status === 0, "fixture grants dispatch authority"); + + const executed = parseJson(runNode(cewpCli, [ + "supervise", "execute", runId, "--yes", "--timeout", "20", "--json", + ], repoRoot, { + env: { + ...fake.env, + CEWP_FAKE_CODEX_EXPECT_MODEL: "gpt-test-explicit", + CEWP_FAKE_CODEX_EXPECT_EFFORT: "high", + }, + }), "execute with approved effort policy"); + const attempt = executed.data.run.tasks[0].attempts[0]; + assert(attempt.codex.taskClass === "demanding-implementation", "dispatch evidence retains the approved task class"); + assert(attempt.codex.effectiveModel.status === "known", "explicit dispatch model becomes known evidence"); + assert(attempt.codex.effectiveModel.value === "gpt-test-explicit", "effective model matches the approved override"); + assert(attempt.codex.effectiveEffort.status === "known", "explicit dispatch effort becomes known evidence"); + assert(attempt.codex.effectiveEffort.value === "high", "effective effort matches the approved override"); + + const verified = parseJson(runNode(cewpCli, [ + "supervise", "verify", runId, "--timeout", "20", "--json", + ], repoRoot), "verify explicit-effort checkpoint"); + assert(verified.data.run.status === "checkpoint-complete", "review setup retains the verification gate"); + parseJson(runNode(cewpCli, [ + "supervise", "effort", runId, + "--operation", "reviewer", + "--task-class", "high-effort-independent-review", + "--model", "gpt-test-reviewer", + "--effort", "xhigh", + "--yes", "--json", + ], repoRoot), "approve reviewer effort policy"); + const reviewed = parseJson(runNode(cewpCli, [ + "supervise", "review", runId, "--yes", "--timeout", "20", "--json", + ], repoRoot, { + env: { + ...fake.env, + CEWP_FAKE_CODEX_EXPECT_MODEL: "gpt-test-reviewer", + CEWP_FAKE_CODEX_EXPECT_EFFORT: "xhigh", + }, + }), "review with approved effort policy"); + assert(reviewed.data.run.reviewer.codex.taskClass === "high-effort-independent-review", "review evidence retains its task class"); + assert(reviewed.data.run.reviewer.codex.effectiveModel.value === "gpt-test-reviewer", "review evidence records the effective model"); + assert(reviewed.data.run.reviewer.codex.effectiveEffort.value === "xhigh", "review evidence records the effective effort"); + } finally { + fs.rmSync(fake.fakeRoot, { recursive: true, force: true }); + cleanupRepo(repoRoot); + } +} + +try { + main(); + runRepairEffortContract(); + runEffortTamperContract(); + runStalePlanRevisionContract(); + console.log("[PASS] Codex task classes never trigger automatic model routing"); +} catch (error) { + console.error("[FAIL] Codex effort policy contract"); + console.error(error && error.stack ? error.stack : error); + process.exitCode = 1; +} diff --git a/tests/harness/lib/fake-adapter.js b/tests/harness/lib/fake-adapter.js index 53438f2..bc528ac 100644 --- a/tests/harness/lib/fake-adapter.js +++ b/tests/harness/lib/fake-adapter.js @@ -38,6 +38,17 @@ if (args[0] !== "exec") { process.exit(2); } +const expectedModel = process.env.CEWP_FAKE_CODEX_EXPECT_MODEL; +const expectedEffort = process.env.CEWP_FAKE_CODEX_EXPECT_EFFORT; +if (expectedModel && valueAfter("--model") !== expectedModel) { + console.error("fake codex did not receive the approved --model override"); + process.exit(3); +} +if (expectedEffort && !args.includes('model_reasoning_effort="' + expectedEffort + '"')) { + console.error("fake codex did not receive the approved reasoning-effort override"); + process.exit(3); +} + function valueAfter(flag) { const index = args.indexOf(flag); return index === -1 ? undefined : args[index + 1]; From 7b3cb9dcc529470ce4bbff9cdf3e01af5fb990f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tuncay=20=C3=96lmez?= Date: Wed, 22 Jul 2026 11:00:00 +0300 Subject: [PATCH 05/40] Document supervised effort policy --- docs/supervised-workflow.md | 20 +++++++++++++++++++ package.json | 5 +++-- .../skills/run-supervised-checkpoint/SKILL.md | 6 +++--- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/docs/supervised-workflow.md b/docs/supervised-workflow.md index 10e57ad..b68cc6b 100644 --- a/docs/supervised-workflow.md +++ b/docs/supervised-workflow.md @@ -56,6 +56,26 @@ Changing the goal, scope, checks, or stopping conditions creates a new plan revi ## 3. Execute And Verify +An optional Codex-specific sidecar can assign an explicit task class without changing the +provider-neutral workflow or run schemas: + +```bash +cewp supervise effort \ + --operation implementation \ + --task-class demanding-implementation \ + --model \ + --effort high \ + --yes +``` + +Supported operations are `implementation`, `repair`, and `reviewer`. Supported task classes are +`fast-exploration`, `demanding-implementation`, and `high-effort-independent-review`. A task class +never selects a model or reasoning effort automatically. Every initial selection or change requires +`--yes`, creates a revision with an operator-approval digest, and fails closed when the sidecar's +approved selection digest no longer matches. Model and effort remain `unknown` when omitted. Explicit selections become known +effective evidence only after supported structured turn-completion usage is received; an early CLI, +model, or host rejection leaves them unknown. + Managed execution is an advanced local action: ```bash diff --git a/package.json b/package.json index 811b8db..1c5014f 100644 --- a/package.json +++ b/package.json @@ -50,7 +50,7 @@ }, "scripts": { "test": "npm run test:contracts && npm run smoke", - "test:contracts": "npm run test:init-install && npm run test:adapter-profile && npm run test:doctor-json && npm run test:operator-json && npm run test:hook-output && npm run test:skill-format && npm run test:ownership-gates && npm run test:fixtures && npm run test:plugin-package && npm run test:supervised-intake && npm run test:supervised-proposal && npm run test:supervised-controls && npm run test:supervised-execution && npm run test:supervised-review && npm run test:supervised-failure && npm run test:supervised-linear-resume && npm run test:supervised-demo && npm run test:workflow-definition && npm run test:workflow-compiler && npm run test:workflow-proposal && npm run test:workflow-scheduler && npm run test:workflow-worker-matrix && npm run test:workflow-result && npm run test:workflow-failure-result && npm run test:workflow-state-machine && npm run test:workflow-interventions && npm run test:workflow-failure-matrix && npm run test:workflow-budget && npm run test:workflow-progress && npm run test:workflow-review && npm run test:workflow-checkpoint-review && npm run test:workflow-lifecycle && npm run test:workflow-revision && npm run test:workflow-migration && npm run test:workflow-templates && npm run test:workflow-docs && npm run test:workflow-release && npm run test:integration-capabilities && npm run test:integration-binding && npm run test:integration-observation && npm run test:native-goal-events", + "test:contracts": "npm run test:init-install && npm run test:adapter-profile && npm run test:doctor-json && npm run test:operator-json && npm run test:hook-output && npm run test:skill-format && npm run test:ownership-gates && npm run test:fixtures && npm run test:plugin-package && npm run test:supervised-intake && npm run test:supervised-proposal && npm run test:supervised-controls && npm run test:supervised-execution && npm run test:supervised-review && npm run test:supervised-failure && npm run test:supervised-linear-resume && npm run test:supervised-demo && npm run test:workflow-definition && npm run test:workflow-compiler && npm run test:workflow-proposal && npm run test:workflow-scheduler && npm run test:workflow-worker-matrix && npm run test:workflow-result && npm run test:workflow-failure-result && npm run test:workflow-state-machine && npm run test:workflow-interventions && npm run test:workflow-failure-matrix && npm run test:workflow-budget && npm run test:workflow-progress && npm run test:workflow-review && npm run test:workflow-checkpoint-review && npm run test:workflow-lifecycle && npm run test:workflow-revision && npm run test:workflow-migration && npm run test:workflow-templates && npm run test:workflow-docs && npm run test:workflow-release && npm run test:integration-capabilities && npm run test:integration-binding && npm run test:integration-effort-policy && npm run test:integration-observation && npm run test:native-goal-events", "test:init-install": "node tests/contracts/init-install.js", "test:adapter-profile": "node tests/contracts/adapter-profile.js", "test:doctor-json": "node tests/contracts/doctor-json.js", @@ -90,6 +90,7 @@ "test:workflow-release": "node tests/contracts/workflow-release.js", "test:integration-capabilities": "node tests/contracts/integration-capabilities.js", "test:integration-binding": "node tests/contracts/integration-binding.js", + "test:integration-effort-policy": "node tests/contracts/integration-effort-policy.js", "test:integration-observation": "node tests/contracts/integration-observation.js", "test:native-goal-events": "node tests/contracts/native-goal-events.js", "demo:supervised": "node src/demo/supervised.js", @@ -98,7 +99,7 @@ "probe:codex-app-server": "node tests/capabilities/codex-app-server.js", "smoke": "node tests/harness/run-smoke.js", "check:cli": "node ./bin/cewp.js --help", - "check:syntax": "node --check ./bin/cewp.js && node --check ./src/demo/fake-codex.js && node --check ./src/demo/supervised.js && node --check ./src/integration/capabilities.js && node --check ./src/integration/binding.js && node --check ./src/integration/observation.js && node --check ./src/integration/native-goal.js && node --check ./src/skills/install.js && node --check ./src/skills/format.js && node --check ./src/skills/doctor-report.js && node --check ./src/run/ownership.js && node --check ./src/run/control-gates.js && node --check ./src/supervise/budget.js && node --check ./src/supervise/cli.js && node --check ./src/supervise/commands.js && node --check ./src/supervise/controls.js && node --check ./src/supervise/execution.js && node --check ./src/supervise/profiles.js && node --check ./src/supervise/receipt.js && node --check ./src/supervise/review.js && node --check ./src/supervise/state.js && node --check ./src/supervise/test-authoring.js && node --check ./src/supervise/verification.js && node --check ./src/workflow/cli.js && node --check ./src/workflow/definition.js && node --check ./src/workflow/graph.js && node --check ./src/workflow/result.js && node --check ./src/workflow/scheduler.js && node --check ./src/workflow/source.js && node --check ./src/workflow/state.js && node --check ./src/workflow/transitions.js && node --check ./tests/contracts/init-install.js && node --check ./tests/contracts/adapter-profile.js && node --check ./tests/contracts/doctor-json.js && node --check ./tests/contracts/operator-json.js && node --check ./tests/contracts/hook-output.js && node --check ./tests/contracts/skill-format.js && node --check ./tests/contracts/ownership-gates.js && node --check ./tests/contracts/deterministic-fixtures.js && node --check ./tests/contracts/plugin-package.js && node --check ./tests/contracts/supervised-intake.js && node --check ./tests/contracts/supervised-proposal.js && node --check ./tests/contracts/supervised-controls.js && node --check ./tests/contracts/supervised-execution.js && node --check ./tests/contracts/supervised-review.js && node --check ./tests/contracts/supervised-failure.js && node --check ./tests/contracts/supervised-linear-resume.js && node --check ./tests/contracts/supervised-demo.js && node --check ./tests/contracts/integration-capabilities.js && node --check ./tests/contracts/integration-binding.js && node --check ./tests/contracts/integration-observation.js && node --check ./tests/contracts/native-goal-events.js && node --check ./tests/contracts/workflow-definition.js && node --check ./tests/contracts/workflow-proposal.js && node --check ./tests/contracts/workflow-scheduler.js && node --check ./tests/contracts/workflow-result.js && node --check ./tests/contracts/workflow-state-machine.js && node --check ./tests/contracts/workflow-interventions.js && node --check ./tests/capabilities/clean-install.js && node --check ./tests/capabilities/plugin-lifecycle.js && node --check ./tests/capabilities/codex-app-server.js && node --check ./tests/capabilities/fixtures/deny-shell-hook.js", + "check:syntax": "node --check ./bin/cewp.js && node --check ./src/demo/fake-codex.js && node --check ./src/demo/supervised.js && node --check ./src/integration/capabilities.js && node --check ./src/integration/binding.js && node --check ./src/integration/effort-policy.js && node --check ./src/integration/observation.js && node --check ./src/integration/native-goal.js && node --check ./src/skills/install.js && node --check ./src/skills/format.js && node --check ./src/skills/doctor-report.js && node --check ./src/run/ownership.js && node --check ./src/run/control-gates.js && node --check ./src/supervise/budget.js && node --check ./src/supervise/cli.js && node --check ./src/supervise/commands.js && node --check ./src/supervise/controls.js && node --check ./src/supervise/execution.js && node --check ./src/supervise/profiles.js && node --check ./src/supervise/receipt.js && node --check ./src/supervise/review.js && node --check ./src/supervise/state.js && node --check ./src/supervise/test-authoring.js && node --check ./src/supervise/verification.js && node --check ./src/workflow/cli.js && node --check ./src/workflow/definition.js && node --check ./src/workflow/graph.js && node --check ./src/workflow/result.js && node --check ./src/workflow/scheduler.js && node --check ./src/workflow/source.js && node --check ./src/workflow/state.js && node --check ./src/workflow/transitions.js && node --check ./tests/contracts/init-install.js && node --check ./tests/contracts/adapter-profile.js && node --check ./tests/contracts/doctor-json.js && node --check ./tests/contracts/operator-json.js && node --check ./tests/contracts/hook-output.js && node --check ./tests/contracts/skill-format.js && node --check ./tests/contracts/ownership-gates.js && node --check ./tests/contracts/deterministic-fixtures.js && node --check ./tests/contracts/plugin-package.js && node --check ./tests/contracts/supervised-intake.js && node --check ./tests/contracts/supervised-proposal.js && node --check ./tests/contracts/supervised-controls.js && node --check ./tests/contracts/supervised-execution.js && node --check ./tests/contracts/supervised-review.js && node --check ./tests/contracts/supervised-failure.js && node --check ./tests/contracts/supervised-linear-resume.js && node --check ./tests/contracts/supervised-demo.js && node --check ./tests/contracts/integration-capabilities.js && node --check ./tests/contracts/integration-binding.js && node --check ./tests/contracts/integration-effort-policy.js && node --check ./tests/contracts/integration-observation.js && node --check ./tests/contracts/native-goal-events.js && node --check ./tests/contracts/workflow-definition.js && node --check ./tests/contracts/workflow-proposal.js && node --check ./tests/contracts/workflow-scheduler.js && node --check ./tests/contracts/workflow-result.js && node --check ./tests/contracts/workflow-state-machine.js && node --check ./tests/contracts/workflow-interventions.js && node --check ./tests/capabilities/clean-install.js && node --check ./tests/capabilities/plugin-lifecycle.js && node --check ./tests/capabilities/codex-app-server.js && node --check ./tests/capabilities/fixtures/deny-shell-hook.js", "check:workflow-syntax": "node --check ./src/workflow/budget.js && node --check ./src/workflow/compiler.js && node --check ./src/workflow/migration.js && node --check ./src/workflow/progress.js && node --check ./src/workflow/proposal.js && node --check ./src/workflow/review.js && node --check ./src/workflow/revision.js && node --check ./src/workflow/templates.js && node --check ./tests/contracts/workflow-budget.js && node --check ./tests/contracts/workflow-checkpoint-review.js && node --check ./tests/contracts/workflow-compiler.js && node --check ./tests/contracts/workflow-docs.js && node --check ./tests/contracts/workflow-failure-matrix.js && node --check ./tests/contracts/workflow-failure-result.js && node --check ./tests/contracts/workflow-migration.js && node --check ./tests/contracts/workflow-progress.js && node --check ./tests/contracts/workflow-release.js && node --check ./tests/contracts/workflow-review.js && node --check ./tests/contracts/workflow-lifecycle.js && node --check ./tests/contracts/workflow-revision.js && node --check ./tests/contracts/workflow-templates.js && node --check ./tests/contracts/workflow-worker-matrix.js", "check": "npm run check:syntax && npm run check:workflow-syntax && npm test", "pack:dry-run": "npm pack --dry-run" diff --git a/plugins/cewp/skills/run-supervised-checkpoint/SKILL.md b/plugins/cewp/skills/run-supervised-checkpoint/SKILL.md index 717e208..4350cc0 100644 --- a/plugins/cewp/skills/run-supervised-checkpoint/SKILL.md +++ b/plugins/cewp/skills/run-supervised-checkpoint/SKILL.md @@ -11,9 +11,9 @@ Use canonical JSON from `cewp supervise`; generated Markdown is a view, not muta 2. Perform no more than one CEWP-controlled model operation per invocation. Local verification commands do not count as model operations, but their approved limits still apply. 3. Follow only the transition matching canonical state: - `proposed`: show the plan and run `cewp supervise approve --yes --json` only after explicit approval. Stop before dispatch. - - `approved/ready`: run `cewp supervise execute --yes --json` only after explicit execution approval. If dispatch reaches `verifying`, run `cewp supervise verify --json`; then stop at the verified, repair, paused, or blocked result. - - `needs-repair/repair-ready`: explain the failure signature and remaining repair allocation. Run `cewp supervise retry --yes --json` only when the user explicitly chooses retry, then run local verification and stop. - - `checkpoint-complete/verified`: offer final review or a safe pause for one manually bounded next checkpoint. For final review, record `cewp supervise continue --json`, run `cewp supervise review --yes --json`, and stop at the reviewer decision. For more work, hand off to the resume workflow without dispatching another model operation. + - `approved/ready`: if the operator explicitly requested a Codex task class, model, or effort, record it first with `cewp supervise effort --operation implementation --task-class [--model ] [--effort ] --yes --json`. Never infer model or effort from the class. Then run `cewp supervise execute --yes --json` only after explicit execution approval. If dispatch reaches `verifying`, run `cewp supervise verify --json`; then stop at the verified, repair, paused, or blocked result. + - `needs-repair/repair-ready`: explain the failure signature and remaining repair allocation. Record any explicitly requested repair-specific Codex selection through `cewp supervise effort ... --operation repair ... --yes --json`; do not reuse or change implementation settings implicitly. Run `cewp supervise retry --yes --json` only when the user explicitly chooses retry, then run local verification and stop. + - `checkpoint-complete/verified`: offer final review or a safe pause for one manually bounded next checkpoint. For final review, record any explicitly requested independent-review Codex selection through `cewp supervise effort ... --operation reviewer ... --yes --json`, record `cewp supervise continue --json`, run `cewp supervise review --yes --json`, and stop at the reviewer decision. For more work, hand off to the resume workflow without dispatching another model operation. - `review-passed`: run `cewp supervise receipt --json` to preview the receipt. Do not finalize in the same step unless the user already gave explicit finalize intent after seeing equivalent receipt facts. - `ready-to-finalize`: run `cewp supervise finalize --yes --json` only after explicit finalization approval. - any `paused-*` or `blocked`: do not dispatch. Present Core recovery actions and hand off to the resume workflow. From 72760a6d860116df9bf360453a9751c81508f408 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tuncay=20=C3=96lmez?= Date: Wed, 22 Jul 2026 11:30:00 +0300 Subject: [PATCH 06/40] Add trusted subagent hook evidence --- bin/cewp.js | 8 +- src/cli/parse.js | 20 +- src/cli/usage.js | 2 + src/integration/cli.js | 66 +++++ src/integration/hook-evidence.js | 402 +++++++++++++++++++++++++++++++ 5 files changed, 495 insertions(+), 3 deletions(-) create mode 100644 src/integration/cli.js create mode 100644 src/integration/hook-evidence.js diff --git a/bin/cewp.js b/bin/cewp.js index a6b91b2..c5c2af2 100644 --- a/bin/cewp.js +++ b/bin/cewp.js @@ -34,6 +34,7 @@ const { init } = require("../src/skills/install"); const { list, doctor } = require("../src/skills/status"); const { runSupervise } = require("../src/supervise/cli"); const { runWorkflow } = require("../src/workflow/cli"); +const { runIntegration } = require("../src/integration/cli"); const { printHuman: printSupervisedDemo, runSupervisedDemo } = require("../src/demo/supervised"); function runDemo(options) { @@ -209,12 +210,17 @@ async function main() { return; } + if (args.command === "integration") { + runIntegration(args); + return; + } + if (args.command === "demo") { runDemo(args); return; } - if (!["init", "list", "doctor", "policy", "run", "supervise", "workflow", "demo"].includes(args.command)) { + if (!["init", "list", "doctor", "policy", "run", "supervise", "workflow", "integration", "demo"].includes(args.command)) { throw new Error(`Unsupported command: ${args.command}`); } } catch (error) { diff --git a/src/cli/parse.js b/src/cli/parse.js index 43cd61c..e39fae8 100644 --- a/src/cli/parse.js +++ b/src/cli/parse.js @@ -97,7 +97,7 @@ function parseArgs(argv) { return args; } - const optionStart = ["run", "supervise", "workflow", "demo"].includes(args.command) ? 2 : 1; + const optionStart = ["run", "supervise", "workflow", "integration", "demo"].includes(args.command) ? 2 : 1; for (let index = optionStart; index < argv.length; index += 1) { const arg = argv[index]; @@ -132,6 +132,16 @@ function parseArgs(argv) { continue; } + if (args.command === "integration" && args.subcommand === "hooks" && index === 2) { + args.action = arg; + continue; + } + + if (args.command === "integration" && args.subcommand === "hooks" && index === 3 && !arg.startsWith("--")) { + args.workflowRunId = arg; + continue; + } + if (args.command === "workflow" && args.subcommand === "validate" && index === 2 && !arg.startsWith("--")) { args.definitionFile = arg; continue; @@ -446,7 +456,7 @@ function parseArgs(argv) { continue; } - if (["doctor", "run", "supervise", "workflow", "demo"].includes(args.command) && arg === "--json") { + if (["doctor", "run", "supervise", "workflow", "integration", "demo"].includes(args.command) && arg === "--json") { args.json = true; continue; } @@ -461,6 +471,12 @@ function parseArgs(argv) { continue; } + + if (args.command === "integration" && arg === "--yes") { + args.yes = true; + continue; + } + if (args.command === "supervise" && arg === "--allow-test-authoring") { args.allowTestAuthoring = true; continue; diff --git a/src/cli/usage.js b/src/cli/usage.js index d63d86b..7a7aa4a 100644 --- a/src/cli/usage.js +++ b/src/cli/usage.js @@ -17,6 +17,8 @@ Usage: cewp workflow result --task --result --yes [--json] cewp workflow review --result --yes [--json] cewp workflow finalize --yes [--json] + cewp integration hooks approve --yes [--json] + cewp integration hooks status [--json] cewp workflow revise --proposal [--from ] [--json] cewp workflow apply-revision --proposal --digest --yes [--json] cewp workflow migrate [--digest --yes] [--json] diff --git a/src/integration/cli.js b/src/integration/cli.js new file mode 100644 index 0000000..26a10b9 --- /dev/null +++ b/src/integration/cli.js @@ -0,0 +1,66 @@ +"use strict"; + +const fs = require("node:fs"); +const { + approveCodexHookTrust, + inspectCodexHookTrust, + recordSubagentHookEvent, +} = require("./hook-evidence"); + +function outputJson(command, data) { + console.log(JSON.stringify({ + schemaVersion: "operator-json/v1", + command, + generatedAt: new Date().toISOString(), + data, + warnings: [], + }, null, 2)); +} + +function runIntegration(options = {}) { + if (options.subcommand === "hooks" && options.action === "ingest") { + const raw = fs.readFileSync(0, "utf8"); + if (Buffer.byteLength(raw, "utf8") > 64 * 1024) { + throw new Error("Codex hook input exceeds 65536 bytes."); + } + const input = JSON.parse(raw || "{}"); + recordSubagentHookEvent({ input }); + console.log("{}"); + return; + } + + if (options.subcommand === "hooks" && options.action === "approve") { + if (!options.workflowRunId) throw new Error("integration hooks approve requires a workflow run id."); + const result = approveCodexHookTrust({ + repoRoot: process.cwd(), + runId: options.workflowRunId, + yes: options.yes, + }); + if (options.json) outputJson("integration.hooks.approve", result); + else { + console.log("CEWP Codex hook evidence approved"); + console.log(`Run ID: ${result.trust.runId}`); + console.log(`Bundle: ${result.trust.bundleDigest}`); + console.log("Next: open /hooks and review the current definition"); + } + return; + } + if (options.subcommand === "hooks" && options.action === "status") { + if (!options.workflowRunId) throw new Error("integration hooks status requires a workflow run id."); + const result = inspectCodexHookTrust({ + repoRoot: process.cwd(), + runId: options.workflowRunId, + }); + if (options.json) outputJson("integration.hooks.status", result); + else { + console.log("CEWP Codex hook evidence status"); + console.log(`Run ID: ${result.runId}`); + console.log(`Active: ${result.active ? "yes" : "no"}`); + result.warnings.forEach((warning) => console.log(`Warning: ${warning.code}: ${warning.message}`)); + } + return; + } + throw new Error(`Unsupported integration command: ${options.subcommand || "missing"}.`); +} + +module.exports = { runIntegration }; diff --git a/src/integration/hook-evidence.js b/src/integration/hook-evidence.js new file mode 100644 index 0000000..c20dd76 --- /dev/null +++ b/src/integration/hook-evidence.js @@ -0,0 +1,402 @@ +"use strict"; + +const childProcess = require("node:child_process"); +const crypto = require("node:crypto"); +const fs = require("node:fs"); +const path = require("node:path"); +const { loadWorkflowRun } = require("../workflow/state"); +const { writeJsonAtomic } = require("../workflow/state"); + +const CODEX_HOOK_TRUST_SCHEMA_VERSION = "codex-hook-trust/v1"; +const CODEX_HOOK_CONTRACT_VERSION = "codex-hooks-doc/2026-07-22"; +const HOOK_BUNDLE_FILES = Object.freeze([ + "hooks/hooks.json", + "hooks/capture-subagent.js", +]); +const SUBAGENT_HOOK_EVIDENCE_SCHEMA_VERSION = "subagent-hook-evidence/v1"; +const SUBAGENT_HOOK_EVENTS = Object.freeze({ + SubagentStart: "subagent-started", + SubagentStop: "subagent-stopped", +}); + +function sha256(value) { + return `sha256:${crypto.createHash("sha256").update(value).digest("hex")}`; +} + +function pluginRoot() { + if (process.env.PLUGIN_ROOT) return path.resolve(process.env.PLUGIN_ROOT); + return path.resolve(__dirname, "..", "..", "plugins", "cewp"); +} + +function inspectHookBundle(root = pluginRoot()) { + const files = HOOK_BUNDLE_FILES.map((relativePath) => { + const filePath = path.join(root, ...relativePath.split("/")); + if (!fs.existsSync(filePath)) throw new Error(`Codex hook bundle file is missing: ${relativePath}.`); + const content = fs.readFileSync(filePath); + return { path: relativePath, digest: sha256(content), content }; + }); + const bundle = Buffer.concat(files.flatMap((file) => [ + Buffer.from(`${file.path}\0`, "utf8"), + file.content, + Buffer.from("\0", "utf8"), + ])); + return { + digest: sha256(bundle), + files: files.map(({ path: relativePath, digest }) => ({ path: relativePath, digest })), + }; +} + +function detectCodexVersion(options = {}) { + if (process.env.CEWP_HOOK_CODEX_VERSION) return process.env.CEWP_HOOK_CODEX_VERSION.trim(); + const command = options.command || "codex"; + const result = childProcess.spawnSync(command, ["--version"], { + encoding: "utf8", + windowsHide: true, + timeout: 10000, + }); + if (result.status !== 0 || !String(result.stdout || "").trim()) { + throw new Error("Codex hook approval requires a successful local `codex --version` probe."); + } + return String(result.stdout).trim(); +} + +function detectCewpVersion() { + const packagePath = path.resolve(__dirname, "..", "..", "package.json"); + if (!fs.existsSync(packagePath)) { + throw new Error("Codex hook evidence requires the complete CEWP package, including package.json."); + } + const packageJson = JSON.parse(fs.readFileSync(packagePath, "utf8")); + if (typeof packageJson.version !== "string" || packageJson.version.trim().length === 0) { + throw new Error("Codex hook evidence could not determine the CEWP runtime version."); + } + return packageJson.version.trim(); +} + +function trustPath(found) { + return path.join(found.runRoot, "integration", "codex-hook-trust.json"); +} + +function activationPath(repoRoot) { + return path.join(repoRoot, ".cewp", "integration", "active-hook-trust.json"); +} + +function boundedText(value, label, maximum, optional = false) { + if (optional && (value === null || value === undefined)) return null; + if (typeof value !== "string" || value.trim().length === 0) { + throw new Error(`Invalid Codex subagent hook event: ${label} is required.`); + } + const text = value.trim(); + if (text.length > maximum || /[\u0000-\u0008\u000b\u000c\u000e-\u001f]/.test(text)) { + throw new Error(`Invalid Codex subagent hook event: ${label} is too long or contains control characters.`); + } + return text; +} + +function findActivatedRepoRoot(startPath) { + let current = path.resolve(startPath); + while (true) { + if (fs.existsSync(activationPath(current))) return current; + const parent = path.dirname(current); + if (parent === current) return null; + current = parent; + } +} + +function validateActiveTrust(repoRoot, currentCodexVersion, root = pluginRoot(), currentCewpVersion) { + const activePath = activationPath(repoRoot); + if (!fs.existsSync(activePath)) return null; + const active = JSON.parse(fs.readFileSync(activePath, "utf8")); + if (active.schemaVersion !== "codex-hook-activation/v1") { + throw new Error("Codex hook evidence activation is malformed; approve the hook bundle again."); + } + const found = loadWorkflowRun(repoRoot, active.runId); + const filePath = trustPath(found); + if (!fs.existsSync(filePath)) throw new Error("Codex hook trust receipt is missing; approve the hook bundle again."); + const trust = JSON.parse(fs.readFileSync(filePath, "utf8")); + if (trust.schemaVersion !== CODEX_HOOK_TRUST_SCHEMA_VERSION || trust.provider !== "codex") { + throw new Error("Codex hook trust receipt is malformed; approve the hook bundle again."); + } + const selection = { + runId: trust.runId, + workflowRevision: trust.workflowRevision, + workflowDigest: trust.workflowDigest, + cewpVersion: trust.cewpVersion, + codexVersion: trust.codexVersion, + hookContractVersion: trust.hookContractVersion, + bundleDigest: trust.bundleDigest, + }; + const approvalDigest = sha256(JSON.stringify(selection)); + if ( + !trust.approval + || trust.approval.kind !== "operator" + || trust.approval.digest !== approvalDigest + || active.trustDigest !== approvalDigest + ) { + throw new Error("Codex hook trust approval digest changed; review and approve the hook bundle again."); + } + const currentBundle = inspectHookBundle(root); + if (trust.bundleDigest !== currentBundle.digest) { + throw new Error("Codex hook definition drift detected; review and approve the current bundle again."); + } + if (trust.codexVersion !== currentCodexVersion) { + throw new Error(`Codex hook version drift detected: approved ${trust.codexVersion}, observed ${currentCodexVersion}.`); + } + const observedCewpVersion = currentCewpVersion || detectCewpVersion(); + if (trust.cewpVersion !== observedCewpVersion) { + throw new Error(`CEWP hook runtime drift detected: approved ${trust.cewpVersion}, observed ${observedCewpVersion}.`); + } + if (trust.hookContractVersion !== CODEX_HOOK_CONTRACT_VERSION) { + throw new Error("Codex hook contract drift detected; review and approve the current contract again."); + } + if ( + trust.runId !== found.run.runId + || trust.workflowRevision !== found.run.workflow.revision + || trust.workflowDigest !== found.run.workflow.digest + ) { + throw new Error("Codex hook trust is stale for the current workflow revision."); + } + return { active, found, trust }; +} + +function recordSubagentHookEvent(options = {}) { + const input = options.input; + if (!input || typeof input !== "object" || Array.isArray(input)) { + throw new Error("Invalid Codex subagent hook event: expected one JSON object."); + } + const eventCwd = boundedText(input.cwd, "cwd", 4096); + const repoRoot = findActivatedRepoRoot(options.repoRoot || eventCwd); + if (!repoRoot) return { recorded: false, reason: "not-activated" }; + const resolvedCwd = path.resolve(eventCwd); + const relativeCwd = path.relative(repoRoot, resolvedCwd); + if (relativeCwd.startsWith("..") || path.isAbsolute(relativeCwd)) { + throw new Error("Invalid Codex subagent hook event: cwd is outside the activated repository."); + } + const currentCodexVersion = options.codexVersion || detectCodexVersion(options); + const active = validateActiveTrust(repoRoot, currentCodexVersion, options.pluginRoot, options.cewpVersion); + if (!active) return { recorded: false, reason: "not-activated" }; + const hookEventName = boundedText(input.hook_event_name, "hook_event_name", 64); + const type = SUBAGENT_HOOK_EVENTS[hookEventName]; + if (!type) throw new Error(`Invalid Codex subagent hook event: unsupported event ${hookEventName}.`); + const agentId = boundedText(input.agent_id, "agent_id", 512); + const parentSessionId = boundedText(input.session_id, "session_id", 512); + const parentTurnId = boundedText(input.turn_id, "turn_id", 512); + const agentType = boundedText(input.agent_type, "agent_type", 128); + const model = boundedText(input.model, "model", 256); + const permissionMode = boundedText(input.permission_mode, "permission_mode", 64); + const summaryValue = hookEventName === "SubagentStop" + ? boundedText(input.last_assistant_message, "last_assistant_message", 4000, true) + : null; + const observedAt = (options.now || new Date()).toISOString(); + const evidence = { + schemaVersion: SUBAGENT_HOOK_EVIDENCE_SCHEMA_VERSION, + eventId: sha256(JSON.stringify({ + runId: active.found.run.runId, + hookEventName, + parentSessionId, + parentTurnId, + agentId, + observedAt, + })), + type, + observedAt, + workflow: { + runId: active.found.run.runId, + revision: active.found.run.workflow.revision, + digest: active.found.run.workflow.digest, + }, + source: { + path: "plugin-hook", + evidenceClass: "observed", + codexVersion: currentCodexVersion, + cewpVersion: active.trust.cewpVersion, + hookContractVersion: CODEX_HOOK_CONTRACT_VERSION, + bundleDigest: active.trust.bundleDigest, + trustApprovalDigest: active.trust.approval.digest, + }, + references: { + agentId, + agentType, + parentSessionId, + parentTurnId, + agentThreadId: { status: "unknown", value: null, reason: "not-exposed-by-documented-hook-input" }, + }, + context: { + model, + permissionMode, + workingDirectory: path.relative(repoRoot, resolvedCwd).replace(/\\/g, "/") || ".", + }, + summary: summaryValue + ? { status: "observed", value: summaryValue } + : { status: "unknown", value: null }, + claims: { + coreEnforcement: false, + opensCoreGates: false, + transcriptRead: false, + }, + }; + const ledgerPath = path.join(active.found.runRoot, "integration", "subagent-hook-evidence.jsonl"); + fs.mkdirSync(path.dirname(ledgerPath), { recursive: true }); + fs.appendFileSync(ledgerPath, `${JSON.stringify(evidence)}\n`); + return { recorded: true, evidence }; +} + +function approveCodexHookTrust(options = {}) { + if (!options.yes) { + throw new Error("Codex hook evidence requires explicit operator approval with --yes after reviewing the bundle and `/hooks`."); + } + const found = loadWorkflowRun(options.repoRoot || process.cwd(), options.runId); + const bundle = inspectHookBundle(options.pluginRoot); + const codexVersion = options.codexVersion || detectCodexVersion(options); + const approvedAt = (options.now || new Date()).toISOString(); + const selection = { + runId: found.run.runId, + workflowRevision: found.run.workflow.revision, + workflowDigest: found.run.workflow.digest, + cewpVersion: options.cewpVersion || detectCewpVersion(), + codexVersion, + hookContractVersion: CODEX_HOOK_CONTRACT_VERSION, + bundleDigest: bundle.digest, + }; + const trust = { + schemaVersion: CODEX_HOOK_TRUST_SCHEMA_VERSION, + provider: "codex", + ...selection, + bundleFiles: bundle.files, + approvedAt, + approval: { + kind: "operator", + digest: sha256(JSON.stringify(selection)), + }, + hostTrust: { + required: true, + status: "pending-review", + reviewCommand: "/hooks", + }, + provenance: { + source: "official-documentation", + url: "https://learn.chatgpt.com/docs/hooks", + }, + claims: { + coreEnforcement: false, + opensCoreGates: false, + transcriptRead: false, + }, + }; + const filePath = trustPath(found); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + writeJsonAtomic(filePath, trust); + const activePath = activationPath(found.repoRoot); + fs.mkdirSync(path.dirname(activePath), { recursive: true }); + writeJsonAtomic(activePath, { + schemaVersion: "codex-hook-activation/v1", + runId: found.run.runId, + trustDigest: trust.approval.digest, + activatedAt: approvedAt, + }); + fs.appendFileSync(path.join(found.runRoot, "events.jsonl"), `${JSON.stringify({ + schemaVersion: "workflow-event/v1", + timestamp: approvedAt, + type: "hook-evidence-approved", + runId: found.run.runId, + revision: found.run.workflow.revision, + actor: "operator", + approvalDigest: trust.approval.digest, + })}\n`); + return { + trust, + nextAction: { + kind: "host-hook-review", + command: "/hooks", + reason: "Codex separately reviews and trusts the exact current plugin hook definition.", + }, + }; +} + +function classifyCompatibilityWarning(error) { + const message = error && error.message ? error.message : String(error); + const code = /Codex hook version drift/.test(message) + ? "codex-version-drift" + : /definition drift/.test(message) + ? "hook-definition-drift" + : /runtime drift/.test(message) + ? "cewp-version-drift" + : /contract drift/.test(message) + ? "hook-contract-drift" + : /workflow revision|not active/.test(message) + ? "workflow-binding-drift" + : /approval digest|trust receipt/.test(message) + ? "hook-trust-change" + : "hook-evidence-unavailable"; + return { code, message }; +} + +function inspectCodexHookTrust(options = {}) { + const repoRoot = path.resolve(options.repoRoot || process.cwd()); + const found = loadWorkflowRun(repoRoot, options.runId); + const filePath = trustPath(found); + const claims = { + coreEnforcement: false, + opensCoreGates: false, + hostTrustInferred: false, + }; + if (!fs.existsSync(filePath)) { + return { + schemaVersion: "codex-hook-status/v1", + runId: found.run.runId, + compatible: false, + active: false, + trust: null, + warnings: [{ code: "hook-evidence-not-approved", message: "No approved hook evidence bundle exists for this run." }], + fallback: "core-and-conversation-only", + claims, + }; + } + let trust = null; + try { + try { + trust = JSON.parse(fs.readFileSync(filePath, "utf8")); + } catch (error) { + throw new Error(`Codex hook trust receipt is malformed: ${error.message}`); + } + const currentCodexVersion = options.codexVersion || detectCodexVersion(options); + const active = validateActiveTrust(repoRoot, currentCodexVersion, options.pluginRoot, options.cewpVersion); + if (!active || active.found.run.runId !== found.run.runId) { + throw new Error("Codex hook trust is not active for the requested workflow run."); + } + return { + schemaVersion: "codex-hook-status/v1", + runId: found.run.runId, + compatible: true, + active: true, + trust, + warnings: [], + fallback: null, + claims, + }; + } catch (error) { + return { + schemaVersion: "codex-hook-status/v1", + runId: found.run.runId, + compatible: false, + active: false, + trust, + warnings: [classifyCompatibilityWarning(error)], + fallback: "core-and-conversation-only", + claims, + }; + } +} + +module.exports = { + CODEX_HOOK_CONTRACT_VERSION, + CODEX_HOOK_TRUST_SCHEMA_VERSION, + SUBAGENT_HOOK_EVIDENCE_SCHEMA_VERSION, + approveCodexHookTrust, + detectCewpVersion, + detectCodexVersion, + findActivatedRepoRoot, + inspectCodexHookTrust, + inspectHookBundle, + recordSubagentHookEvent, + validateActiveTrust, +}; From d2f294453b74fd4ca495859e81455f109dd97bbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tuncay=20=C3=96lmez?= Date: Wed, 22 Jul 2026 12:00:00 +0300 Subject: [PATCH 07/40] Package trusted subagent capture hooks --- docs/codex-capability-matrix.md | 3 ++ docs/known-limitations.md | 2 +- docs/skill-plugin-compatibility.md | 4 +-- package.json | 5 +-- plugins/cewp/.codex-plugin/plugin.json | 1 + plugins/cewp/README.md | 16 +++++++-- plugins/cewp/hooks/capture-subagent.js | 49 ++++++++++++++++++++++++++ plugins/cewp/hooks/hooks.json | 31 ++++++++++++++++ 8 files changed, 104 insertions(+), 7 deletions(-) create mode 100644 plugins/cewp/hooks/capture-subagent.js create mode 100644 plugins/cewp/hooks/hooks.json diff --git a/docs/codex-capability-matrix.md b/docs/codex-capability-matrix.md index c6ec9f3..3001d03 100644 --- a/docs/codex-capability-matrix.md +++ b/docs/codex-capability-matrix.md @@ -47,6 +47,8 @@ Schema presence does not prove that a plugin can attach to the desktop app's exi | Desktop notifications | host-specific | The host owns documented notification behavior and settings. CEWP has no arbitrary notification category. | | Hook `statusMessage` | supported | Official hook configuration exposes it as transient handler status. | | Hook `systemMessage` | supported | Official hook output exposes it as a UI or event-stream warning. | +| `SubagentStart`/`SubagentStop` evidence | supported, opt-in | The plugin records only documented parent session/turn, agent id/type, permission/model context, and bounded stop summary after separate CEWP approval and host `/hooks` trust. The documented input exposes no subagent thread id, so CEWP preserves it as `unknown`. | +| Hook trust and version drift | supported | `npm run test:integration-hook-evidence` binds the exact bundle, Codex version, CEWP runtime, hook contract, and workflow revision. Drift or malformed input emits a warning, appends no trusted evidence, and leaves Core gates unchanged. | | `PreToolUse` deny output | supported | The deterministic fixture emits the documented `permissionDecision: deny` shape and is covered by `npm run test:hook-output`. | | `PreToolUse` as complete enforcement | unavailable | Official docs exclude or limit richer shell and non-MCP paths. A real CLI 0.137.0 Windows probe executed the requested PowerShell command despite the Bash deny hook. Core policy remains authoritative. | | Hook-based instant turn cancellation | unknown | Stop semantics do not establish instantaneous cancellation of an in-flight model or external process. | @@ -115,6 +117,7 @@ npm run probe:codex-app-server npm run test:plugin-lifecycle npm run test:hook-output npm run test:integration-capabilities +npm run test:integration-hook-evidence ``` The nested model probe is intentionally excluded from automated tests because it consumes account usage. Raw account values, credentials, thread ids, and machine-specific paths are not part of this document. diff --git a/docs/known-limitations.md b/docs/known-limitations.md index d17d910..913d3a5 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -10,7 +10,7 @@ CEWP is beta software. These limits are product boundaries, not hidden roadmap p - ChatGPT subscription credit impact and host-internal retries or compaction remain `unknown` without a supported machine-readable contract. - Numeric usage estimates stay unavailable until enough comparable local runs exist. When available, they are ranges with confidence and sample basis, never point promises. - File-level test-authoring enforcement recognizes common test directories and filename conventions. It cannot prove whether production code contains test-like logic. -- The plugin contributes skills only. It does not provide MCP tools, hooks, an Apps SDK card, or an App Server client. +- The plugin contributes skills and an optional review-required `SubagentStart`/`SubagentStop` evidence hook. The hook cannot expose a subagent thread id, does not read transcripts, and is never a Core enforcement boundary. Local MCP, an Apps SDK card, and an App Server client are not yet shipped. - Experimental OpenCode execution remains optional and outside the supervised golden path. Binary/version availability does not prove authentication or model readiness. - Manual is a non-executing handoff adapter. Claude, Gemini, Hermes, and other providers are not implemented. - Supervised worktree cleanup automation is not shipped; rollback is available for owned unverified work, and terminal evidence is retained for deliberate inspection/removal. diff --git a/docs/skill-plugin-compatibility.md b/docs/skill-plugin-compatibility.md index 1a06e95..a6f9ac6 100644 --- a/docs/skill-plugin-compatibility.md +++ b/docs/skill-plugin-compatibility.md @@ -1,6 +1,6 @@ # Skill And Plugin Compatibility -Observed: 2026-07-16 +Observed: 2026-07-22 CEWP's ten bundled skills use the current Codex skill shape: a required `SKILL.md` with `name` and `description`, plus optional `scripts/`, `references/`, `assets/`, and `agents/`. `agents/openai.yaml` is accepted as optional UI, invocation-policy, and dependency metadata. CEWP no longer treats these official components as forbidden. @@ -23,7 +23,7 @@ The Phase 9 plugin follows the official boundary: - `.codex-plugin/plugin.json` is the required manifest and the only file under `.codex-plugin/`. - `skills/`, `hooks/`, `.mcp.json`, `.app.json`, and `assets/` live at the plugin root and use `./`-prefixed contained paths. -- Installing or enabling a plugin does not trust its bundled hooks. CEWP hooks remain optional until the user reviews and trusts the current definition. +- Installing or enabling a plugin does not trust its bundled hooks. CEWP declares one contained subagent-evidence bundle, requires a run-bound operator approval, and still directs the user to `/hooks` to review and trust the exact current host definition. Bundle, Codex, CEWP runtime, hook-contract, or workflow-revision drift disables trusted evidence and falls back to Core plus conversation output. - A repo marketplace lives at `.agents/plugins/marketplace.json`; npm remains the source of the CEWP Core CLI/runtime. - MCP and Apps SDK components are optional projections. The plugin skeleton and golden path cannot depend on them until their versioned capability tests pass. diff --git a/package.json b/package.json index 1c5014f..7c6d52b 100644 --- a/package.json +++ b/package.json @@ -50,7 +50,7 @@ }, "scripts": { "test": "npm run test:contracts && npm run smoke", - "test:contracts": "npm run test:init-install && npm run test:adapter-profile && npm run test:doctor-json && npm run test:operator-json && npm run test:hook-output && npm run test:skill-format && npm run test:ownership-gates && npm run test:fixtures && npm run test:plugin-package && npm run test:supervised-intake && npm run test:supervised-proposal && npm run test:supervised-controls && npm run test:supervised-execution && npm run test:supervised-review && npm run test:supervised-failure && npm run test:supervised-linear-resume && npm run test:supervised-demo && npm run test:workflow-definition && npm run test:workflow-compiler && npm run test:workflow-proposal && npm run test:workflow-scheduler && npm run test:workflow-worker-matrix && npm run test:workflow-result && npm run test:workflow-failure-result && npm run test:workflow-state-machine && npm run test:workflow-interventions && npm run test:workflow-failure-matrix && npm run test:workflow-budget && npm run test:workflow-progress && npm run test:workflow-review && npm run test:workflow-checkpoint-review && npm run test:workflow-lifecycle && npm run test:workflow-revision && npm run test:workflow-migration && npm run test:workflow-templates && npm run test:workflow-docs && npm run test:workflow-release && npm run test:integration-capabilities && npm run test:integration-binding && npm run test:integration-effort-policy && npm run test:integration-observation && npm run test:native-goal-events", + "test:contracts": "npm run test:init-install && npm run test:adapter-profile && npm run test:doctor-json && npm run test:operator-json && npm run test:hook-output && npm run test:skill-format && npm run test:ownership-gates && npm run test:fixtures && npm run test:plugin-package && npm run test:supervised-intake && npm run test:supervised-proposal && npm run test:supervised-controls && npm run test:supervised-execution && npm run test:supervised-review && npm run test:supervised-failure && npm run test:supervised-linear-resume && npm run test:supervised-demo && npm run test:workflow-definition && npm run test:workflow-compiler && npm run test:workflow-proposal && npm run test:workflow-scheduler && npm run test:workflow-worker-matrix && npm run test:workflow-result && npm run test:workflow-failure-result && npm run test:workflow-state-machine && npm run test:workflow-interventions && npm run test:workflow-failure-matrix && npm run test:workflow-budget && npm run test:workflow-progress && npm run test:workflow-review && npm run test:workflow-checkpoint-review && npm run test:workflow-lifecycle && npm run test:workflow-revision && npm run test:workflow-migration && npm run test:workflow-templates && npm run test:workflow-docs && npm run test:workflow-release && npm run test:integration-capabilities && npm run test:integration-binding && npm run test:integration-effort-policy && npm run test:integration-hook-evidence && npm run test:integration-observation && npm run test:native-goal-events", "test:init-install": "node tests/contracts/init-install.js", "test:adapter-profile": "node tests/contracts/adapter-profile.js", "test:doctor-json": "node tests/contracts/doctor-json.js", @@ -91,6 +91,7 @@ "test:integration-capabilities": "node tests/contracts/integration-capabilities.js", "test:integration-binding": "node tests/contracts/integration-binding.js", "test:integration-effort-policy": "node tests/contracts/integration-effort-policy.js", + "test:integration-hook-evidence": "node tests/contracts/integration-hook-evidence.js", "test:integration-observation": "node tests/contracts/integration-observation.js", "test:native-goal-events": "node tests/contracts/native-goal-events.js", "demo:supervised": "node src/demo/supervised.js", @@ -99,7 +100,7 @@ "probe:codex-app-server": "node tests/capabilities/codex-app-server.js", "smoke": "node tests/harness/run-smoke.js", "check:cli": "node ./bin/cewp.js --help", - "check:syntax": "node --check ./bin/cewp.js && node --check ./src/demo/fake-codex.js && node --check ./src/demo/supervised.js && node --check ./src/integration/capabilities.js && node --check ./src/integration/binding.js && node --check ./src/integration/effort-policy.js && node --check ./src/integration/observation.js && node --check ./src/integration/native-goal.js && node --check ./src/skills/install.js && node --check ./src/skills/format.js && node --check ./src/skills/doctor-report.js && node --check ./src/run/ownership.js && node --check ./src/run/control-gates.js && node --check ./src/supervise/budget.js && node --check ./src/supervise/cli.js && node --check ./src/supervise/commands.js && node --check ./src/supervise/controls.js && node --check ./src/supervise/execution.js && node --check ./src/supervise/profiles.js && node --check ./src/supervise/receipt.js && node --check ./src/supervise/review.js && node --check ./src/supervise/state.js && node --check ./src/supervise/test-authoring.js && node --check ./src/supervise/verification.js && node --check ./src/workflow/cli.js && node --check ./src/workflow/definition.js && node --check ./src/workflow/graph.js && node --check ./src/workflow/result.js && node --check ./src/workflow/scheduler.js && node --check ./src/workflow/source.js && node --check ./src/workflow/state.js && node --check ./src/workflow/transitions.js && node --check ./tests/contracts/init-install.js && node --check ./tests/contracts/adapter-profile.js && node --check ./tests/contracts/doctor-json.js && node --check ./tests/contracts/operator-json.js && node --check ./tests/contracts/hook-output.js && node --check ./tests/contracts/skill-format.js && node --check ./tests/contracts/ownership-gates.js && node --check ./tests/contracts/deterministic-fixtures.js && node --check ./tests/contracts/plugin-package.js && node --check ./tests/contracts/supervised-intake.js && node --check ./tests/contracts/supervised-proposal.js && node --check ./tests/contracts/supervised-controls.js && node --check ./tests/contracts/supervised-execution.js && node --check ./tests/contracts/supervised-review.js && node --check ./tests/contracts/supervised-failure.js && node --check ./tests/contracts/supervised-linear-resume.js && node --check ./tests/contracts/supervised-demo.js && node --check ./tests/contracts/integration-capabilities.js && node --check ./tests/contracts/integration-binding.js && node --check ./tests/contracts/integration-effort-policy.js && node --check ./tests/contracts/integration-observation.js && node --check ./tests/contracts/native-goal-events.js && node --check ./tests/contracts/workflow-definition.js && node --check ./tests/contracts/workflow-proposal.js && node --check ./tests/contracts/workflow-scheduler.js && node --check ./tests/contracts/workflow-result.js && node --check ./tests/contracts/workflow-state-machine.js && node --check ./tests/contracts/workflow-interventions.js && node --check ./tests/capabilities/clean-install.js && node --check ./tests/capabilities/plugin-lifecycle.js && node --check ./tests/capabilities/codex-app-server.js && node --check ./tests/capabilities/fixtures/deny-shell-hook.js", + "check:syntax": "node --check ./bin/cewp.js && node --check ./src/demo/fake-codex.js && node --check ./src/demo/supervised.js && node --check ./src/integration/capabilities.js && node --check ./src/integration/binding.js && node --check ./src/integration/cli.js && node --check ./src/integration/effort-policy.js && node --check ./src/integration/hook-evidence.js && node --check ./src/integration/observation.js && node --check ./src/integration/native-goal.js && node --check ./src/skills/install.js && node --check ./src/skills/format.js && node --check ./src/skills/doctor-report.js && node --check ./src/run/ownership.js && node --check ./src/run/control-gates.js && node --check ./src/supervise/budget.js && node --check ./src/supervise/cli.js && node --check ./src/supervise/commands.js && node --check ./src/supervise/controls.js && node --check ./src/supervise/execution.js && node --check ./src/supervise/profiles.js && node --check ./src/supervise/receipt.js && node --check ./src/supervise/review.js && node --check ./src/supervise/state.js && node --check ./src/supervise/test-authoring.js && node --check ./src/supervise/verification.js && node --check ./src/workflow/cli.js && node --check ./src/workflow/definition.js && node --check ./src/workflow/graph.js && node --check ./src/workflow/result.js && node --check ./src/workflow/scheduler.js && node --check ./src/workflow/source.js && node --check ./src/workflow/state.js && node --check ./src/workflow/transitions.js && node --check ./plugins/cewp/hooks/capture-subagent.js && node --check ./tests/contracts/init-install.js && node --check ./tests/contracts/adapter-profile.js && node --check ./tests/contracts/doctor-json.js && node --check ./tests/contracts/operator-json.js && node --check ./tests/contracts/hook-output.js && node --check ./tests/contracts/skill-format.js && node --check ./tests/contracts/ownership-gates.js && node --check ./tests/contracts/deterministic-fixtures.js && node --check ./tests/contracts/plugin-package.js && node --check ./tests/contracts/supervised-intake.js && node --check ./tests/contracts/supervised-proposal.js && node --check ./tests/contracts/supervised-controls.js && node --check ./tests/contracts/supervised-execution.js && node --check ./tests/contracts/supervised-review.js && node --check ./tests/contracts/supervised-failure.js && node --check ./tests/contracts/supervised-linear-resume.js && node --check ./tests/contracts/supervised-demo.js && node --check ./tests/contracts/integration-capabilities.js && node --check ./tests/contracts/integration-binding.js && node --check ./tests/contracts/integration-effort-policy.js && node --check ./tests/contracts/integration-hook-evidence.js && node --check ./tests/contracts/integration-observation.js && node --check ./tests/contracts/native-goal-events.js && node --check ./tests/contracts/workflow-definition.js && node --check ./tests/contracts/workflow-proposal.js && node --check ./tests/contracts/workflow-scheduler.js && node --check ./tests/contracts/workflow-result.js && node --check ./tests/contracts/workflow-state-machine.js && node --check ./tests/contracts/workflow-interventions.js && node --check ./tests/capabilities/clean-install.js && node --check ./tests/capabilities/plugin-lifecycle.js && node --check ./tests/capabilities/codex-app-server.js && node --check ./tests/capabilities/fixtures/deny-shell-hook.js", "check:workflow-syntax": "node --check ./src/workflow/budget.js && node --check ./src/workflow/compiler.js && node --check ./src/workflow/migration.js && node --check ./src/workflow/progress.js && node --check ./src/workflow/proposal.js && node --check ./src/workflow/review.js && node --check ./src/workflow/revision.js && node --check ./src/workflow/templates.js && node --check ./tests/contracts/workflow-budget.js && node --check ./tests/contracts/workflow-checkpoint-review.js && node --check ./tests/contracts/workflow-compiler.js && node --check ./tests/contracts/workflow-docs.js && node --check ./tests/contracts/workflow-failure-matrix.js && node --check ./tests/contracts/workflow-failure-result.js && node --check ./tests/contracts/workflow-migration.js && node --check ./tests/contracts/workflow-progress.js && node --check ./tests/contracts/workflow-release.js && node --check ./tests/contracts/workflow-review.js && node --check ./tests/contracts/workflow-lifecycle.js && node --check ./tests/contracts/workflow-revision.js && node --check ./tests/contracts/workflow-templates.js && node --check ./tests/contracts/workflow-worker-matrix.js", "check": "npm run check:syntax && npm run check:workflow-syntax && npm test", "pack:dry-run": "npm pack --dry-run" diff --git a/plugins/cewp/.codex-plugin/plugin.json b/plugins/cewp/.codex-plugin/plugin.json index b04e299..e87c827 100644 --- a/plugins/cewp/.codex-plugin/plugin.json +++ b/plugins/cewp/.codex-plugin/plugin.json @@ -16,6 +16,7 @@ "evidence" ], "skills": "./skills/", + "hooks": "./hooks/hooks.json", "interface": { "displayName": "CEWP", "shortDescription": "Long-running goals without blind runs", diff --git a/plugins/cewp/README.md b/plugins/cewp/README.md index 5fba658..5c3e509 100644 --- a/plugins/cewp/README.md +++ b/plugins/cewp/README.md @@ -8,7 +8,19 @@ It ships exactly three entry skills: - `run-supervised-checkpoint`: execute one controlled model operation and follow Core gates. - `resume-supervised-run`: inspect or recover canonical state without silently restarting work. -The plugin does not attach to the ChatGPT desktop app's private thread, automate native goals, inject persistent UI, expose hidden host usage, execute the optional OpenCode adapter, or add another provider. Phase 9 uses one selected pair: `managed` owner with the `codex-exec` backend. +The plugin does not attach to the ChatGPT desktop app's private thread, automate native goals, inject persistent UI, expose hidden host usage, execute the optional OpenCode adapter, or add another provider. The managed path uses one selected pair: `managed` owner with the `codex-exec` backend. + +## Optional Subagent Evidence + +The plugin declares one `SubagentStart`/`SubagentStop` hook bundle. Installation or enablement does not trust it. For a selected workflow run, first inspect the bundle and activate the CEWP-side binding: + +```bash +cewp integration hooks approve --yes --json +``` + +Then open `/hooks` in Codex, review the exact current definition, and decide whether to trust it. `cewp integration hooks status --json` reports bundle, Codex, CEWP runtime, hook-contract, and workflow-revision drift. A changed or malformed source produces a warning and no trusted evidence. + +The hook records bounded parent session/turn references, the documented subagent id/type, and the stop summary in a provider-specific sidecar. The documented hook input does not expose a subagent thread id, so that field remains `unknown`. The handler never reads transcript files, never blocks or continues a subagent, and never opens a policy, verification, review, or finalization gate. Install from the CEWP source marketplace: @@ -17,4 +29,4 @@ codex plugin marketplace add /path/to/Codex-Engineering-Workflow-Pack codex plugin add cewp@cewp-local ``` -The npm package supplies the `cewp` runtime used by these skills. Run `cewp doctor --json` before the first supervised checkpoint. +The npm package supplies the `cewp` runtime used by the skills and optional evidence handler. Run `cewp doctor --json` before the first supervised checkpoint. diff --git a/plugins/cewp/hooks/capture-subagent.js b/plugins/cewp/hooks/capture-subagent.js new file mode 100644 index 0000000..7d0b9ed --- /dev/null +++ b/plugins/cewp/hooks/capture-subagent.js @@ -0,0 +1,49 @@ +#!/usr/bin/env node +"use strict"; + +const childProcess = require("node:child_process"); + +process.stdin.setEncoding("utf8"); +let input = ""; +let oversized = false; +process.stdin.on("data", (chunk) => { + input += chunk; + if (Buffer.byteLength(input, "utf8") > 64 * 1024) oversized = true; +}); +process.stdin.on("end", () => { + try { + if (oversized) throw new Error("hook input exceeds 65536 bytes"); + JSON.parse(input || "{}"); + const command = process.env.CEWP_HOOK_CLI_COMMAND || (process.platform === "win32" ? "cewp.cmd" : "cewp"); + const prefixArgs = process.env.CEWP_HOOK_CLI_PREFIX_ARGS + ? JSON.parse(process.env.CEWP_HOOK_CLI_PREFIX_ARGS) + : []; + if (!Array.isArray(prefixArgs) || prefixArgs.some((value) => typeof value !== "string")) { + throw new Error("invalid CEWP hook CLI prefix configuration"); + } + const result = childProcess.spawnSync(command, [ + ...prefixArgs, + "integration", "hooks", "ingest", + ], { + cwd: process.cwd(), + input, + encoding: "utf8", + windowsHide: true, + timeout: 15000, + shell: process.platform === "win32" && !process.env.CEWP_HOOK_CLI_COMMAND, + }); + if (result.error) throw result.error; + if (result.status !== 0) { + const reason = String(result.stderr || "CEWP hook ingestion failed") + .replace(/\s+/g, " ") + .trim() + .slice(0, 1000); + throw new Error(reason); + } + process.stdout.write("{}\n"); + } catch (error) { + process.stdout.write(`${JSON.stringify({ + systemMessage: `CEWP hook evidence unavailable: ${error.message} Core gates remain unchanged.`, + })}\n`); + } +}); diff --git a/plugins/cewp/hooks/hooks.json b/plugins/cewp/hooks/hooks.json new file mode 100644 index 0000000..ef858e0 --- /dev/null +++ b/plugins/cewp/hooks/hooks.json @@ -0,0 +1,31 @@ +{ + "description": "Optional CEWP subagent lifecycle evidence. CEWP Core gates remain authoritative.", + "hooks": { + "SubagentStart": [ + { + "matcher": "*", + "hooks": [ + { + "type": "command", + "command": "node \"$PLUGIN_ROOT/hooks/capture-subagent.js\"", + "command_windows": "node \"%PLUGIN_ROOT%\\hooks\\capture-subagent.js\"", + "statusMessage": "Recording optional CEWP subagent evidence" + } + ] + } + ], + "SubagentStop": [ + { + "matcher": "*", + "hooks": [ + { + "type": "command", + "command": "node \"$PLUGIN_ROOT/hooks/capture-subagent.js\"", + "command_windows": "node \"%PLUGIN_ROOT%\\hooks\\capture-subagent.js\"", + "statusMessage": "Recording optional CEWP subagent summary" + } + ] + } + ] + } +} From 4fb18e13ac8cffc838957a8f9eeecbdb45ac80cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tuncay=20=C3=96lmez?= Date: Wed, 22 Jul 2026 12:30:00 +0300 Subject: [PATCH 08/40] Test trusted subagent hook evidence --- tests/contracts/integration-hook-evidence.js | 192 +++++++++++++++++++ tests/contracts/plugin-package.js | 11 +- tests/harness/README.md | 1 + 3 files changed, 203 insertions(+), 1 deletion(-) create mode 100644 tests/contracts/integration-hook-evidence.js diff --git a/tests/contracts/integration-hook-evidence.js b/tests/contracts/integration-hook-evidence.js new file mode 100644 index 0000000..72c05d6 --- /dev/null +++ b/tests/contracts/integration-hook-evidence.js @@ -0,0 +1,192 @@ +"use strict"; + +const fs = require("node:fs"); +const childProcess = require("node:child_process"); +const path = require("node:path"); +const { assert } = require("../harness/lib/assertions"); +const { cleanupRepo, makeTempRepo, runNode } = require("../harness/lib/temp-repo"); +const { validDefinition } = require("./workflow-definition"); +const { approveWorkflow } = require("./workflow-scheduler"); +const { recordSubagentHookEvent } = require("../../src/integration/hook-evidence"); + +const cewpCli = path.join(__dirname, "..", "..", "bin", "cewp.js"); + +function assertThrows(action, expected, label) { + let error; + try { + action(); + } catch (caught) { + error = caught; + } + assert(error, `${label}: expected an error`); + assert(expected.test(error.message), `${label}: unexpected error: ${error.message}`); +} + +function main() { + const repoRoot = makeTempRepo("cewp-hook-evidence-"); + try { + const run = approveWorkflow(repoRoot, validDefinition()); + const runPath = path.join(repoRoot, ".cewp", "workflow-runs", run.runId, "run.json"); + const runBefore = fs.readFileSync(runPath, "utf8"); + const refused = runNode(cewpCli, [ + "integration", "hooks", "approve", run.runId, "--json", + ], repoRoot, { + env: { ...process.env, CEWP_HOOK_CODEX_VERSION: "codex-cli 0.200.0" }, + }); + assert(refused.status === 1, "hook trust cannot be activated without explicit --yes approval"); + assert(refused.stderr.includes("explicit operator approval"), "approval refusal explains the trust boundary"); + const approved = runNode(cewpCli, [ + "integration", "hooks", "approve", run.runId, "--yes", "--json", + ], repoRoot, { + env: { ...process.env, CEWP_HOOK_CODEX_VERSION: "codex-cli 0.200.0" }, + }); + assert(approved.status === 0, `hook approval succeeds: ${approved.stderr}`); + const output = JSON.parse(approved.stdout); + assert(output.command === "integration.hooks.approve", "approval identifies the public command"); + assert(output.data.trust.schemaVersion === "codex-hook-trust/v1", "hook trust is versioned"); + assert(output.data.trust.cewpVersion === "0.10.0-beta.0", "approval binds the CEWP runtime version"); + assert(output.data.trust.codexVersion === "codex-cli 0.200.0", "approval binds the observed Codex version"); + assert(/^sha256:[a-f0-9]{64}$/.test(output.data.trust.bundleDigest), "approval binds the exact hook bundle"); + assert(output.data.nextAction.command === "/hooks", "approval still requires the host trust review"); + assert(fs.readFileSync(runPath, "utf8") === runBefore, "hook trust stays outside provider-neutral run state"); + + const inspected = runNode(cewpCli, [ + "integration", "hooks", "status", run.runId, "--json", + ], repoRoot, { + env: { ...process.env, CEWP_HOOK_CODEX_VERSION: "codex-cli 0.200.0" }, + }); + assert(inspected.status === 0, `hook status succeeds: ${inspected.stderr}`); + const status = JSON.parse(inspected.stdout); + assert(status.command === "integration.hooks.status", "status identifies the public inspection command"); + assert(status.data.compatible === true && status.data.active === true, "current approved hook evidence is active"); + assert(status.data.claims.coreEnforcement === false, "status never promotes hook evidence to Core enforcement"); + + const driftStatus = runNode(cewpCli, [ + "integration", "hooks", "status", run.runId, "--json", + ], repoRoot, { + env: { ...process.env, CEWP_HOOK_CODEX_VERSION: "codex-cli 0.201.0" }, + }); + assert(driftStatus.status === 0, `hook drift status remains inspectable: ${driftStatus.stderr}`); + const drift = JSON.parse(driftStatus.stdout).data; + assert(drift.compatible === false && drift.active === false, "version drift disables trusted hook evidence"); + assert(drift.warnings[0].code === "codex-version-drift", "version drift has a stable compatibility code"); + assert(drift.fallback === "core-and-conversation-only", "version drift names the safe fallback"); + + const hookPath = path.join(__dirname, "..", "..", "plugins", "cewp", "hooks", "capture-subagent.js"); + const runHook = (input) => childProcess.spawnSync(process.execPath, [hookPath], { + cwd: repoRoot, + input: JSON.stringify(input), + encoding: "utf8", + windowsHide: true, + env: { + ...process.env, + CEWP_HOOK_CODEX_VERSION: "codex-cli 0.200.0", + CEWP_HOOK_CLI_COMMAND: process.execPath, + CEWP_HOOK_CLI_PREFIX_ARGS: JSON.stringify([cewpCli]), + }, + }); + const common = { + session_id: "parent-session-1", + transcript_path: path.join(repoRoot, "does-not-exist.jsonl"), + cwd: repoRoot, + model: "gpt-test-host", + turn_id: "parent-turn-1", + agent_id: "agent-1", + agent_type: "explorer", + permission_mode: "default", + }; + const started = runHook({ ...common, hook_event_name: "SubagentStart" }); + assert(started.status === 0, `SubagentStart hook succeeds: ${started.stderr}`); + assert(JSON.stringify(JSON.parse(started.stdout)) === "{}", "evidence-only start does not steer the subagent"); + const stopped = runHook({ + ...common, + hook_event_name: "SubagentStop", + agent_transcript_path: path.join(repoRoot, "also-does-not-exist.jsonl"), + stop_hook_active: false, + last_assistant_message: "Inspected the bounded files and found no scope issue.", + }); + assert(stopped.status === 0, `SubagentStop hook succeeds: ${stopped.stderr}`); + assert(JSON.stringify(JSON.parse(stopped.stdout)) === "{}", "evidence-only stop does not continue or block the subagent"); + + const ledgerPath = path.join( + repoRoot, ".cewp", "workflow-runs", run.runId, "integration", "subagent-hook-evidence.jsonl", + ); + const evidence = fs.readFileSync(ledgerPath, "utf8").trim().split("\n").map((line) => JSON.parse(line)); + assert(evidence.length === 2, "start and stop lifecycle evidence are append-only"); + assert(evidence[0].type === "subagent-started" && evidence[1].type === "subagent-stopped", "supported lifecycle types are normalized"); + assert(evidence[1].references.agentId === "agent-1", "documented subagent id is preserved"); + assert(evidence[1].references.parentSessionId === "parent-session-1", "documented parent session is preserved"); + assert(evidence[1].references.agentThreadId.status === "unknown", "an unavailable subagent thread id is never invented"); + assert(evidence[1].summary.value.includes("no scope issue"), "bounded host summary is retained"); + assert(evidence[1].claims.coreEnforcement === false, "hook evidence never claims Core enforcement"); + + const ledgerBeforeDrift = fs.readFileSync(ledgerPath, "utf8"); + const versionDrift = childProcess.spawnSync(process.execPath, [hookPath], { + cwd: repoRoot, + input: JSON.stringify({ ...common, hook_event_name: "SubagentStart", agent_id: "agent-drift" }), + encoding: "utf8", + windowsHide: true, + env: { + ...process.env, + CEWP_HOOK_CODEX_VERSION: "codex-cli 0.201.0", + CEWP_HOOK_CLI_COMMAND: process.execPath, + CEWP_HOOK_CLI_PREFIX_ARGS: JSON.stringify([cewpCli]), + }, + }); + assert(versionDrift.status === 0, "version drift does not break the host lifecycle"); + assert(JSON.parse(versionDrift.stdout).systemMessage.includes("version drift"), "version drift is visible and actionable"); + assert(fs.readFileSync(ledgerPath, "utf8") === ledgerBeforeDrift, "version drift cannot append trusted evidence"); + + const malformed = runHook({ ...common, hook_event_name: "SubagentStart", agent_id: null }); + assert(malformed.status === 0, "malformed hook input fails without breaking the host lifecycle"); + assert(JSON.parse(malformed.stdout).systemMessage.includes("agent_id is required"), "malformed input explains the compatibility failure"); + assert(fs.readFileSync(ledgerPath, "utf8") === ledgerBeforeDrift, "malformed input cannot append evidence"); + + const changedPluginRoot = path.join(repoRoot, "changed-plugin"); + fs.mkdirSync(path.join(changedPluginRoot, "hooks"), { recursive: true }); + fs.copyFileSync( + path.join(__dirname, "..", "..", "plugins", "cewp", "hooks", "hooks.json"), + path.join(changedPluginRoot, "hooks", "hooks.json"), + ); + fs.copyFileSync(hookPath, path.join(changedPluginRoot, "hooks", "capture-subagent.js")); + fs.appendFileSync(path.join(changedPluginRoot, "hooks", "capture-subagent.js"), "\n// changed after review\n"); + assertThrows( + () => recordSubagentHookEvent({ + repoRoot, + input: { ...common, hook_event_name: "SubagentStart", agent_id: "agent-definition-drift" }, + codexVersion: "codex-cli 0.200.0", + pluginRoot: changedPluginRoot, + }), + /definition drift/, + "changed hook definitions require a fresh review", + ); + assert(fs.readFileSync(ledgerPath, "utf8") === ledgerBeforeDrift, "definition drift cannot append trusted evidence"); + assert(fs.readFileSync(runPath, "utf8") === runBefore, "hook failures and observations never mutate Core workflow state"); + + const trustPath = path.join( + repoRoot, ".cewp", "workflow-runs", run.runId, "integration", "codex-hook-trust.json", + ); + fs.writeFileSync(trustPath, "{ malformed trust receipt\n"); + const malformedTrustStatus = runNode(cewpCli, [ + "integration", "hooks", "status", run.runId, "--json", + ], repoRoot, { + env: { ...process.env, CEWP_HOOK_CODEX_VERSION: "codex-cli 0.200.0" }, + }); + assert(malformedTrustStatus.status === 0, "malformed trust remains inspectable through a fail-safe status"); + const malformedTrust = JSON.parse(malformedTrustStatus.stdout).data; + assert(malformedTrust.active === false && malformedTrust.compatible === false, "malformed trust disables evidence"); + assert(malformedTrust.warnings[0].code === "hook-trust-change", "malformed trust has a stable warning code"); + assert(malformedTrust.fallback === "core-and-conversation-only", "malformed trust preserves the safe fallback"); + } finally { + cleanupRepo(repoRoot); + } +} + +try { + main(); + console.log("[PASS] hook evidence is explicitly approved and version-bound"); +} catch (error) { + console.error("[FAIL] hook evidence integration contract"); + console.error(error && error.stack ? error.stack : error); + process.exitCode = 1; +} diff --git a/tests/contracts/plugin-package.js b/tests/contracts/plugin-package.js index a2a4e1e..b1f63e6 100644 --- a/tests/contracts/plugin-package.js +++ b/tests/contracts/plugin-package.js @@ -24,7 +24,16 @@ function runPluginPackageContract() { assert(manifest.skills === "./skills/", "plugin skill path is contained and relative"); assert(manifest.apps === undefined, "plugin does not claim an unbuilt app"); assert(manifest.mcpServers === undefined, "plugin does not claim an unbuilt MCP server"); - assert(manifest.hooks === undefined, "plugin does not enable unreviewed hooks"); + assert(manifest.hooks === "./hooks/hooks.json", "plugin declares one contained reviewable hook bundle"); + const hookConfig = readJson(path.join(repoRoot, "plugins", "cewp", "hooks", "hooks.json")); + assert( + JSON.stringify(Object.keys(hookConfig.hooks).sort()) === JSON.stringify(["SubagentStart", "SubagentStop"]), + "plugin hooks are limited to subagent evidence events", + ); + assert( + fs.existsSync(path.join(repoRoot, "plugins", "cewp", "hooks", "capture-subagent.js")), + "declared hook handler exists", + ); assert( fs.existsSync(path.join(repoRoot, "plugins", "cewp", "assets", "cewp.svg")), "plugin asset exists", diff --git a/tests/harness/README.md b/tests/harness/README.md index cc20b65..945ed1c 100644 --- a/tests/harness/README.md +++ b/tests/harness/README.md @@ -25,6 +25,7 @@ npm run test:skill-format npm run test:ownership-gates npm run test:fixtures npm run test:plugin-package +npm run test:integration-hook-evidence ``` `npm test` runs these focused contracts before the broader smoke lifecycle. From 9c3589d0f896948835759177e306eea94e79b2f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tuncay=20=C3=96lmez?= Date: Wed, 22 Jul 2026 13:00:00 +0300 Subject: [PATCH 09/40] Add gated local MCP Core tools --- bin/cewp-mcp.js | 7 ++ src/mcp/server.js | 103 +++++++++++++++++++++++++ src/mcp/tools.js | 188 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 298 insertions(+) create mode 100644 bin/cewp-mcp.js create mode 100644 src/mcp/server.js create mode 100644 src/mcp/tools.js diff --git a/bin/cewp-mcp.js b/bin/cewp-mcp.js new file mode 100644 index 0000000..a3b0ce8 --- /dev/null +++ b/bin/cewp-mcp.js @@ -0,0 +1,7 @@ +#!/usr/bin/env node + +"use strict"; + +const { runStdio } = require("../src/mcp/server"); + +runStdio(); diff --git a/src/mcp/server.js b/src/mcp/server.js new file mode 100644 index 0000000..b83de69 --- /dev/null +++ b/src/mcp/server.js @@ -0,0 +1,103 @@ +"use strict"; + +const readline = require("node:readline"); +const { TOOLS, callTool } = require("./tools"); + +const PROTOCOL_VERSIONS = new Set(["2025-11-25", "2025-06-18", "2025-03-26", "2024-11-05"]); + +function packageVersion() { + try { + return require("../../package.json").version; + } catch { + return "unknown"; + } +} + +function response(id, result) { + return { jsonrpc: "2.0", id, result }; +} + +function rpcError(id, code, message) { + return { jsonrpc: "2.0", id, error: { code, message } }; +} + +function toolResult(value) { + return { + content: [{ type: "text", text: JSON.stringify(value) }], + structuredContent: value, + isError: false, + }; +} + +function toolError(error) { + return { + content: [{ type: "text", text: error && error.message ? error.message : String(error) }], + isError: true, + }; +} + +function createMcpSession(options = {}) { + const repoRoot = options.repoRoot || process.cwd(); + let initialized = false; + + function handle(message) { + if (!message || message.jsonrpc !== "2.0" || typeof message.method !== "string") { + return rpcError(message && message.id !== undefined ? message.id : null, -32600, "Invalid JSON-RPC request."); + } + if (message.method === "notifications/initialized" || message.method.startsWith("notifications/")) { + return null; + } + if (message.method === "initialize") { + const requested = message.params && message.params.protocolVersion; + const protocolVersion = PROTOCOL_VERSIONS.has(requested) ? requested : "2025-11-25"; + initialized = true; + return response(message.id, { + protocolVersion, + capabilities: { tools: { listChanged: false } }, + serverInfo: { + name: "cewp-local-core", + version: packageVersion(), + description: "Local CEWP Core tools with supervised approval and evidence gates.", + }, + instructions: "Inspect before mutating. Explicit confirmation does not bypass CEWP Core state, ownership, scope, policy, budget, verification, or reviewer gates.", + }); + } + if (!initialized) return rpcError(message.id, -32002, "MCP session is not initialized."); + if (message.method === "ping") return response(message.id, {}); + if (message.method === "tools/list") return response(message.id, { tools: TOOLS }); + if (message.method === "tools/call") { + const params = message.params; + if (!params || typeof params.name !== "string") { + return rpcError(message.id, -32602, "tools/call requires a tool name."); + } + try { + return response(message.id, toolResult(callTool(params.name, params.arguments || {}, { repoRoot }))); + } catch (error) { + if (error && error.code === -32602) return rpcError(message.id, -32602, error.message); + return response(message.id, toolError(error)); + } + } + return rpcError(message.id, -32601, `Method not found: ${message.method}`); + } + + return { handle }; +} + +function runStdio(options = {}) { + const session = createMcpSession(options); + const lines = readline.createInterface({ input: process.stdin, crlfDelay: Infinity }); + lines.on("line", (line) => { + if (!line.trim()) return; + let message; + try { + message = JSON.parse(line); + } catch { + process.stdout.write(`${JSON.stringify(rpcError(null, -32700, "Parse error."))}\n`); + return; + } + const result = session.handle(message); + if (result) process.stdout.write(`${JSON.stringify(result)}\n`); + }); +} + +module.exports = { createMcpSession, runStdio }; diff --git a/src/mcp/tools.js b/src/mcp/tools.js new file mode 100644 index 0000000..08e8032 --- /dev/null +++ b/src/mcp/tools.js @@ -0,0 +1,188 @@ +"use strict"; + +const { + approveSupervisedRun, + createProposedRun, + inspectSupervisedRun, +} = require("../supervise/state"); +const { retrySupervisedCheckpoint } = require("../supervise/execution"); +const { verifySupervisedCheckpoint } = require("../supervise/verification"); +const { finalizeSupervisedRun } = require("../supervise/receipt"); +const { runSupervisedControl } = require("../supervise/controls"); + +const stringArray = { type: "array", items: { type: "string" } }; +const runId = { type: "string", minLength: 1, description: "CEWP supervised run ID." }; +const confirm = { + type: "boolean", + description: "Explicit operator confirmation after reviewing the relevant CEWP evidence.", +}; + +function objectSchema(properties, required = []) { + return { type: "object", properties, required, additionalProperties: false }; +} + +const TOOLS = [ + { + name: "cewp_create", + title: "Create CEWP run", + description: "Create a proposed bounded supervised run in the current repository. Does not approve or execute it.", + inputSchema: objectSchema({ + goal: { type: "string", minLength: 1 }, + scopes: stringArray, + verificationCommands: stringArray, + fullVerificationCommands: stringArray, + stoppingConditions: stringArray, + assurance: { type: "string", enum: ["prototype", "standard", "critical"] }, + testAuthoring: { type: "string", enum: ["auto", "ask", "never"] }, + }, ["goal", "scopes", "verificationCommands", "stoppingConditions"]), + annotations: { destructiveHint: false, idempotentHint: false, openWorldHint: false }, + }, + { + name: "cewp_inspect", + title: "Inspect CEWP run", + description: "Inspect canonical state and the next allowed action, refreshing the derived progress file.", + inputSchema: objectSchema({ runId }, ["runId"]), + annotations: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: false }, + }, + { + name: "cewp_approve", + title: "Approve CEWP run", + description: "Approve the current proposed plan revision after explicit operator confirmation. Does not execute it.", + inputSchema: objectSchema({ runId, confirm, allowTestAuthoring: { type: "boolean" } }, ["runId", "confirm"]), + annotations: { destructiveHint: false, idempotentHint: false, openWorldHint: false }, + }, + { + name: "cewp_continue", + title: "Continue CEWP run", + description: "Record operator continuation only after the Core confirms a verified checkpoint.", + inputSchema: objectSchema({ runId }, ["runId"]), + annotations: { destructiveHint: false, idempotentHint: false, openWorldHint: false }, + }, + { + name: "cewp_retry", + title: "Retry CEWP checkpoint", + description: "Dispatch one bounded managed repair, subject to ownership, policy, effort, scope, and budget gates.", + inputSchema: objectSchema({ runId, confirm, timeoutSeconds: { type: "integer", minimum: 1 } }, ["runId", "confirm"]), + annotations: { destructiveHint: true, idempotentHint: false, openWorldHint: false }, + }, + { + name: "cewp_revise", + title: "Revise CEWP plan", + description: "Revise an allowed unstarted or completed checkpoint and invalidate prior approval as required by Core.", + inputSchema: objectSchema({ + runId, + goal: { type: "string", minLength: 1 }, + scopes: stringArray, + verificationCommands: stringArray, + fullVerificationCommands: stringArray, + stoppingConditions: stringArray, + }, ["runId"]), + annotations: { destructiveHint: false, idempotentHint: false, openWorldHint: false }, + }, + { + name: "cewp_verify", + title: "Verify CEWP checkpoint", + description: "Run the approved verification schedule in the managed worktree under Core policy and budget gates.", + inputSchema: objectSchema({ runId, timeoutSeconds: { type: "integer", minimum: 1 } }, ["runId"]), + annotations: { destructiveHint: true, idempotentHint: false, openWorldHint: false }, + }, + { + name: "cewp_finalize", + title: "Finalize CEWP run", + description: "Finalize only after receipt preview, verified evidence, current worktree gates, and independent reviewer PASS.", + inputSchema: objectSchema({ runId, confirm }, ["runId", "confirm"]), + annotations: { destructiveHint: true, idempotentHint: false, openWorldHint: false }, + }, +]; + +const definitions = new Map(TOOLS.map((tool) => [tool.name, tool])); + +function validateArguments(name, args) { + const definition = definitions.get(name); + if (!definition) { + const error = new Error(`Unknown tool: ${name}`); + error.code = -32602; + throw error; + } + if (!args || typeof args !== "object" || Array.isArray(args)) { + const error = new Error(`Tool ${name} arguments must be an object.`); + error.code = -32602; + throw error; + } + const allowed = new Set(Object.keys(definition.inputSchema.properties)); + const unknown = Object.keys(args).filter((key) => !allowed.has(key)); + if (unknown.length > 0) { + const error = new Error(`Tool ${name} received unknown arguments: ${unknown.join(", ")}.`); + error.code = -32602; + throw error; + } + for (const required of definition.inputSchema.required || []) { + if (!(required in args)) { + const error = new Error(`Tool ${name} requires argument ${required}.`); + error.code = -32602; + throw error; + } + } + for (const [key, value] of Object.entries(args)) { + const schema = definition.inputSchema.properties[key]; + const valid = schema.type === "array" + ? Array.isArray(value) && value.every((entry) => typeof entry === "string") + : schema.type === "string" + ? typeof value === "string" && (!schema.minLength || value.length >= schema.minLength) + : schema.type === "boolean" + ? typeof value === "boolean" + : schema.type === "integer" + ? Number.isInteger(value) && (!schema.minimum || value >= schema.minimum) + : true; + if (!valid || (schema.enum && !schema.enum.includes(value))) { + const error = new Error(`Tool ${name} argument ${key} does not match its input schema.`); + error.code = -32602; + throw error; + } + } +} + +function requireConfirmation(name, args) { + if (args.confirm !== true) { + throw new Error(`${name} requires explicit confirmation after reviewing CEWP state and evidence.`); + } +} + +function coreOptions(repoRoot, args) { + return { + ...args, + repoRoot, + scopes: Array.isArray(args.scopes) ? args.scopes : [], + verificationCommands: Array.isArray(args.verificationCommands) ? args.verificationCommands : [], + fullVerificationCommands: Array.isArray(args.fullVerificationCommands) ? args.fullVerificationCommands : [], + stoppingConditions: Array.isArray(args.stoppingConditions) ? args.stoppingConditions : [], + }; +} + +function callTool(name, args, options = {}) { + validateArguments(name, args); + const repoRoot = options.repoRoot || process.cwd(); + const core = coreOptions(repoRoot, args); + if (name === "cewp_create") return createProposedRun(core); + if (name === "cewp_inspect") return inspectSupervisedRun(core); + if (name === "cewp_approve") { + requireConfirmation("approve", args); + return approveSupervisedRun({ ...core, yes: true }); + } + if (name === "cewp_continue") { + return runSupervisedControl({ ...core, subcommand: "continue" }); + } + if (name === "cewp_retry") { + requireConfirmation("retry", args); + return retrySupervisedCheckpoint({ ...core, yes: true }); + } + if (name === "cewp_revise") return runSupervisedControl({ ...core, subcommand: "revise" }); + if (name === "cewp_verify") return verifySupervisedCheckpoint(core); + if (name === "cewp_finalize") { + requireConfirmation("finalize", args); + return finalizeSupervisedRun({ ...core, yes: true }); + } + throw new Error(`Unknown tool: ${name}`); +} + +module.exports = { TOOLS, callTool }; From a8b6b50bd3da812765b870ee081a72a993bcff09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tuncay=20=C3=96lmez?= Date: Wed, 22 Jul 2026 13:30:00 +0300 Subject: [PATCH 10/40] Package local MCP Core integration --- README.md | 9 +++++++-- docs/adr/0005-codex-integration-backend.md | 1 + docs/install.md | 10 ++++++++++ docs/known-limitations.md | 2 +- docs/supervised-workflow.md | 18 ++++++++++++++++++ package.json | 7 +++++-- plugins/cewp/.codex-plugin/plugin.json | 1 + plugins/cewp/.mcp.json | 9 +++++++++ plugins/cewp/README.md | 8 ++++++++ 9 files changed, 60 insertions(+), 5 deletions(-) create mode 100644 plugins/cewp/.mcp.json diff --git a/README.md b/README.md index a3e5826..0864231 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,9 @@ Nothing in that flow merges, pushes, publishes, tags, or creates a release. ## Codex Plugin -The thin plugin contributes exactly three conversational skills. CEWP Core and the CLI remain authoritative. +The thin plugin contributes exactly three conversational skills, a local stdio MCP bridge, and an optional +review-required subagent evidence hook. Every mutating MCP operation delegates to the same CEWP Core used +by the CLI; plugin surfaces do not become execution owners or bypass gates. From a source checkout: @@ -77,7 +79,10 @@ codex plugin add cewp@cewp-local codex plugin list ``` -Then ask Codex to plan a supervised run, run the current checkpoint, or resume an existing run. The plugin does not gain direct access to the host's private thread, native goal lifecycle, billing data, or persistent UI. +Then ask Codex to plan a supervised run, run the current checkpoint, or resume an existing run. The plugin +can expose `cewp_create`, `cewp_inspect`, `cewp_approve`, `cewp_continue`, `cewp_retry`, `cewp_revise`, +`cewp_verify`, and `cewp_finalize` when the package-provided `cewp-mcp` command is on `PATH`. It does not +gain direct access to the host's private thread, native goal lifecycle, billing data, or persistent UI. ## What CEWP Records diff --git a/docs/adr/0005-codex-integration-backend.md b/docs/adr/0005-codex-integration-backend.md index b654a4c..dbf01a9 100644 --- a/docs/adr/0005-codex-integration-backend.md +++ b/docs/adr/0005-codex-integration-backend.md @@ -39,6 +39,7 @@ The Phase 11 review used the current Codex manual and a controlled local probe a - Existing `codex-exec` users keep a stable fallback and one backend per managed checkpoint. - Native goals remain useful without CEWP pretending to control or inspect a private host session. - External agent interfaces can use MCP and `operator-json/v1` without CEWP building a competing terminal or desktop UI. +- The shipped MCP transport is local stdio only, fixes repository scope to process `cwd`, and delegates all eight operations directly to CEWP Core. It is a control surface, not an execution owner or reviewer. - Capability or schema drift produces an explicit compatibility warning and returns to generated-goal or explicit intake. - App Server can be reconsidered later without changing CEWP's provider-neutral workflow and evidence schemas. diff --git a/docs/install.md b/docs/install.md index ab7dd01..89c0a61 100644 --- a/docs/install.md +++ b/docs/install.md @@ -228,6 +228,16 @@ The harness uses temporary repos, exercises Coordinator Mode runtime helpers, an The supervised demo uses a deterministic fake Codex process in a temporary repository. It does not use credentials or start a real provider. +The npm package also installs the plugin-declared local MCP command: + +```bash +cewp-mcp +``` + +It is a stdio protocol process, not an interactive shell command or network server. Codex starts it from +the repository selected for the task. Third-party MCP clients may configure the same command with the +intended repository as `cwd`; the command must be available on `PATH`. + If Codex does not show installed skills, restart or reload Codex and confirm that each skill has: ```txt diff --git a/docs/known-limitations.md b/docs/known-limitations.md index 913d3a5..b03550f 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -10,7 +10,7 @@ CEWP is beta software. These limits are product boundaries, not hidden roadmap p - ChatGPT subscription credit impact and host-internal retries or compaction remain `unknown` without a supported machine-readable contract. - Numeric usage estimates stay unavailable until enough comparable local runs exist. When available, they are ranges with confidence and sample basis, never point promises. - File-level test-authoring enforcement recognizes common test directories and filename conventions. It cannot prove whether production code contains test-like logic. -- The plugin contributes skills and an optional review-required `SubagentStart`/`SubagentStop` evidence hook. The hook cannot expose a subagent thread id, does not read transcripts, and is never a Core enforcement boundary. Local MCP, an Apps SDK card, and an App Server client are not yet shipped. +- The plugin contributes skills, a local stdio MCP bridge, and an optional review-required `SubagentStart`/`SubagentStop` evidence hook. The hook cannot expose a subagent thread id, does not read transcripts, and is never a Core enforcement boundary. MCP exposes only CEWP Core operations and does not attach to native host sessions. An Apps SDK card and App Server client are not shipped. - Experimental OpenCode execution remains optional and outside the supervised golden path. Binary/version availability does not prove authentication or model readiness. - Manual is a non-executing handoff adapter. Claude, Gemini, Hermes, and other providers are not implemented. - Supervised worktree cleanup automation is not shipped; rollback is available for owned unverified work, and terminal evidence is retained for deliberate inspection/removal. diff --git a/docs/supervised-workflow.md b/docs/supervised-workflow.md index b68cc6b..86a25f1 100644 --- a/docs/supervised-workflow.md +++ b/docs/supervised-workflow.md @@ -156,6 +156,24 @@ Generated files include: Editing `progress.md` does not mutate `run.json`. Managed token categories are observed only from valid structured turn events. Host-internal usage remains unknown when the selected boundary does not expose it. +## Local MCP Bridge + +The npm package installs `cewp-mcp`, a local stdio MCP server. The Codex plugin declares it through +`plugins/cewp/.mcp.json`; it opens no network listener and uses the server process working directory as +the fixed repository root. The eight tools are `cewp_create`, `cewp_inspect`, `cewp_approve`, +`cewp_continue`, `cewp_retry`, `cewp_revise`, `cewp_verify`, and `cewp_finalize`. + +The MCP dispatcher imports the same CEWP Core functions used by `cewp supervise`; it does not invoke a +second workflow implementation. CLI `--yes` gates map to a required `confirm: true` argument for approve, +retry, and finalize. All state, execution ownership, policy, effort, scope, budget, verification, receipt, +and independent-review checks remain in Core. MCP host approval is an additional host safety boundary and +never substitutes for those checks. Business-rule failures return structured tool errors; malformed calls +and unknown tools return JSON-RPC protocol errors. + +The plugin MCP entry expects the package-provided `cewp-mcp` binary to be on `PATH`. If a third-party MCP +client cannot discover the plugin manifest, configure a local stdio server with command `cewp-mcp` and set +its working directory to the intended repository. Do not point one server process at multiple repositories. + ## Cleanup `cewp demo supervised` removes its temporary repository and worktree automatically. diff --git a/package.json b/package.json index 7c6d52b..1f1678b 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ }, "bin": { "cewp": "bin/cewp.js", + "cewp-mcp": "bin/cewp-mcp.js", "codex-engineering-workflow-pack": "bin/cewp.js" }, "files": [ @@ -50,7 +51,7 @@ }, "scripts": { "test": "npm run test:contracts && npm run smoke", - "test:contracts": "npm run test:init-install && npm run test:adapter-profile && npm run test:doctor-json && npm run test:operator-json && npm run test:hook-output && npm run test:skill-format && npm run test:ownership-gates && npm run test:fixtures && npm run test:plugin-package && npm run test:supervised-intake && npm run test:supervised-proposal && npm run test:supervised-controls && npm run test:supervised-execution && npm run test:supervised-review && npm run test:supervised-failure && npm run test:supervised-linear-resume && npm run test:supervised-demo && npm run test:workflow-definition && npm run test:workflow-compiler && npm run test:workflow-proposal && npm run test:workflow-scheduler && npm run test:workflow-worker-matrix && npm run test:workflow-result && npm run test:workflow-failure-result && npm run test:workflow-state-machine && npm run test:workflow-interventions && npm run test:workflow-failure-matrix && npm run test:workflow-budget && npm run test:workflow-progress && npm run test:workflow-review && npm run test:workflow-checkpoint-review && npm run test:workflow-lifecycle && npm run test:workflow-revision && npm run test:workflow-migration && npm run test:workflow-templates && npm run test:workflow-docs && npm run test:workflow-release && npm run test:integration-capabilities && npm run test:integration-binding && npm run test:integration-effort-policy && npm run test:integration-hook-evidence && npm run test:integration-observation && npm run test:native-goal-events", + "test:contracts": "npm run test:init-install && npm run test:adapter-profile && npm run test:doctor-json && npm run test:operator-json && npm run test:hook-output && npm run test:skill-format && npm run test:ownership-gates && npm run test:fixtures && npm run test:plugin-package && npm run test:supervised-intake && npm run test:supervised-proposal && npm run test:supervised-controls && npm run test:supervised-execution && npm run test:supervised-review && npm run test:supervised-failure && npm run test:supervised-linear-resume && npm run test:supervised-demo && npm run test:workflow-definition && npm run test:workflow-compiler && npm run test:workflow-proposal && npm run test:workflow-scheduler && npm run test:workflow-worker-matrix && npm run test:workflow-result && npm run test:workflow-failure-result && npm run test:workflow-state-machine && npm run test:workflow-interventions && npm run test:workflow-failure-matrix && npm run test:workflow-budget && npm run test:workflow-progress && npm run test:workflow-review && npm run test:workflow-checkpoint-review && npm run test:workflow-lifecycle && npm run test:workflow-revision && npm run test:workflow-migration && npm run test:workflow-templates && npm run test:workflow-docs && npm run test:workflow-release && npm run test:integration-capabilities && npm run test:integration-binding && npm run test:integration-effort-policy && npm run test:integration-hook-evidence && npm run test:integration-mcp && npm run test:integration-observation && npm run test:native-goal-events", "test:init-install": "node tests/contracts/init-install.js", "test:adapter-profile": "node tests/contracts/adapter-profile.js", "test:doctor-json": "node tests/contracts/doctor-json.js", @@ -92,6 +93,7 @@ "test:integration-binding": "node tests/contracts/integration-binding.js", "test:integration-effort-policy": "node tests/contracts/integration-effort-policy.js", "test:integration-hook-evidence": "node tests/contracts/integration-hook-evidence.js", + "test:integration-mcp": "node tests/contracts/integration-mcp.js", "test:integration-observation": "node tests/contracts/integration-observation.js", "test:native-goal-events": "node tests/contracts/native-goal-events.js", "demo:supervised": "node src/demo/supervised.js", @@ -102,7 +104,8 @@ "check:cli": "node ./bin/cewp.js --help", "check:syntax": "node --check ./bin/cewp.js && node --check ./src/demo/fake-codex.js && node --check ./src/demo/supervised.js && node --check ./src/integration/capabilities.js && node --check ./src/integration/binding.js && node --check ./src/integration/cli.js && node --check ./src/integration/effort-policy.js && node --check ./src/integration/hook-evidence.js && node --check ./src/integration/observation.js && node --check ./src/integration/native-goal.js && node --check ./src/skills/install.js && node --check ./src/skills/format.js && node --check ./src/skills/doctor-report.js && node --check ./src/run/ownership.js && node --check ./src/run/control-gates.js && node --check ./src/supervise/budget.js && node --check ./src/supervise/cli.js && node --check ./src/supervise/commands.js && node --check ./src/supervise/controls.js && node --check ./src/supervise/execution.js && node --check ./src/supervise/profiles.js && node --check ./src/supervise/receipt.js && node --check ./src/supervise/review.js && node --check ./src/supervise/state.js && node --check ./src/supervise/test-authoring.js && node --check ./src/supervise/verification.js && node --check ./src/workflow/cli.js && node --check ./src/workflow/definition.js && node --check ./src/workflow/graph.js && node --check ./src/workflow/result.js && node --check ./src/workflow/scheduler.js && node --check ./src/workflow/source.js && node --check ./src/workflow/state.js && node --check ./src/workflow/transitions.js && node --check ./plugins/cewp/hooks/capture-subagent.js && node --check ./tests/contracts/init-install.js && node --check ./tests/contracts/adapter-profile.js && node --check ./tests/contracts/doctor-json.js && node --check ./tests/contracts/operator-json.js && node --check ./tests/contracts/hook-output.js && node --check ./tests/contracts/skill-format.js && node --check ./tests/contracts/ownership-gates.js && node --check ./tests/contracts/deterministic-fixtures.js && node --check ./tests/contracts/plugin-package.js && node --check ./tests/contracts/supervised-intake.js && node --check ./tests/contracts/supervised-proposal.js && node --check ./tests/contracts/supervised-controls.js && node --check ./tests/contracts/supervised-execution.js && node --check ./tests/contracts/supervised-review.js && node --check ./tests/contracts/supervised-failure.js && node --check ./tests/contracts/supervised-linear-resume.js && node --check ./tests/contracts/supervised-demo.js && node --check ./tests/contracts/integration-capabilities.js && node --check ./tests/contracts/integration-binding.js && node --check ./tests/contracts/integration-effort-policy.js && node --check ./tests/contracts/integration-hook-evidence.js && node --check ./tests/contracts/integration-observation.js && node --check ./tests/contracts/native-goal-events.js && node --check ./tests/contracts/workflow-definition.js && node --check ./tests/contracts/workflow-proposal.js && node --check ./tests/contracts/workflow-scheduler.js && node --check ./tests/contracts/workflow-result.js && node --check ./tests/contracts/workflow-state-machine.js && node --check ./tests/contracts/workflow-interventions.js && node --check ./tests/capabilities/clean-install.js && node --check ./tests/capabilities/plugin-lifecycle.js && node --check ./tests/capabilities/codex-app-server.js && node --check ./tests/capabilities/fixtures/deny-shell-hook.js", "check:workflow-syntax": "node --check ./src/workflow/budget.js && node --check ./src/workflow/compiler.js && node --check ./src/workflow/migration.js && node --check ./src/workflow/progress.js && node --check ./src/workflow/proposal.js && node --check ./src/workflow/review.js && node --check ./src/workflow/revision.js && node --check ./src/workflow/templates.js && node --check ./tests/contracts/workflow-budget.js && node --check ./tests/contracts/workflow-checkpoint-review.js && node --check ./tests/contracts/workflow-compiler.js && node --check ./tests/contracts/workflow-docs.js && node --check ./tests/contracts/workflow-failure-matrix.js && node --check ./tests/contracts/workflow-failure-result.js && node --check ./tests/contracts/workflow-migration.js && node --check ./tests/contracts/workflow-progress.js && node --check ./tests/contracts/workflow-release.js && node --check ./tests/contracts/workflow-review.js && node --check ./tests/contracts/workflow-lifecycle.js && node --check ./tests/contracts/workflow-revision.js && node --check ./tests/contracts/workflow-templates.js && node --check ./tests/contracts/workflow-worker-matrix.js", - "check": "npm run check:syntax && npm run check:workflow-syntax && npm test", + "check:mcp": "node --check ./bin/cewp-mcp.js && node --check ./src/mcp/server.js && node --check ./src/mcp/tools.js && node --check ./tests/contracts/integration-mcp.js", + "check": "npm run check:syntax && npm run check:workflow-syntax && npm run check:mcp && npm test", "pack:dry-run": "npm pack --dry-run" }, "keywords": [ diff --git a/plugins/cewp/.codex-plugin/plugin.json b/plugins/cewp/.codex-plugin/plugin.json index e87c827..e9c6ea0 100644 --- a/plugins/cewp/.codex-plugin/plugin.json +++ b/plugins/cewp/.codex-plugin/plugin.json @@ -16,6 +16,7 @@ "evidence" ], "skills": "./skills/", + "mcpServers": "./.mcp.json", "hooks": "./hooks/hooks.json", "interface": { "displayName": "CEWP", diff --git a/plugins/cewp/.mcp.json b/plugins/cewp/.mcp.json new file mode 100644 index 0000000..7aa9e5b --- /dev/null +++ b/plugins/cewp/.mcp.json @@ -0,0 +1,9 @@ +{ + "mcpServers": { + "cewp": { + "command": "cewp-mcp", + "args": [], + "env": {} + } + } +} diff --git a/plugins/cewp/README.md b/plugins/cewp/README.md index 5c3e509..d6fcf4a 100644 --- a/plugins/cewp/README.md +++ b/plugins/cewp/README.md @@ -10,6 +10,14 @@ It ships exactly three entry skills: The plugin does not attach to the ChatGPT desktop app's private thread, automate native goals, inject persistent UI, expose hidden host usage, execute the optional OpenCode adapter, or add another provider. The managed path uses one selected pair: `managed` owner with the `codex-exec` backend. +## Local MCP Tools + +The plugin declares one local stdio server backed by the npm package's `cewp-mcp` command. It exposes +create, inspect, approve, continue, retry, revise, verify, and finalize as structured tools. The server fixes +repository scope to its working directory and imports the same CEWP Core services as the CLI. MCP host +consent never replaces Core approval, ownership, policy, effort, scope, budget, verification, receipt, or +independent-review gates. + ## Optional Subagent Evidence The plugin declares one `SubagentStart`/`SubagentStop` hook bundle. Installation or enablement does not trust it. For a selected workflow run, first inspect the bundle and activate the CEWP-side binding: From 3d7d0a69ac4505ec83d1a7830a14dbd27261f110 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tuncay=20=C3=96lmez?= Date: Thu, 23 Jul 2026 09:00:00 +0300 Subject: [PATCH 11/40] Test gated local MCP Core tools --- tests/contracts/integration-mcp.js | 103 +++++++++++++++++++++++++++++ tests/contracts/plugin-package.js | 9 ++- 2 files changed, 111 insertions(+), 1 deletion(-) create mode 100644 tests/contracts/integration-mcp.js diff --git a/tests/contracts/integration-mcp.js b/tests/contracts/integration-mcp.js new file mode 100644 index 0000000..a0b0963 --- /dev/null +++ b/tests/contracts/integration-mcp.js @@ -0,0 +1,103 @@ +"use strict"; + +const path = require("node:path"); +const { spawnSync } = require("node:child_process"); +const { assert } = require("../harness/lib/assertions"); +const { cleanupRepo, makeTempRepo } = require("../harness/lib/temp-repo"); + +const mcpBin = path.join(__dirname, "..", "..", "bin", "cewp-mcp.js"); + +function request(id, method, params) { + return JSON.stringify({ jsonrpc: "2.0", id, method, ...(params ? { params } : {}) }); +} + +function runMcp(repoRoot, messages) { + const result = spawnSync(process.execPath, [mcpBin], { + cwd: repoRoot, + encoding: "utf8", + input: `${messages.join("\n")}\n`, + }); + assert(result.status === 0, `MCP server exits cleanly: ${result.stderr}`); + return result.stdout.trim().split(/\r?\n/).filter(Boolean).map((line) => JSON.parse(line)); +} + +function call(id, name, args = {}) { + return request(id, "tools/call", { name, arguments: args }); +} + +function runContract() { + const repoRoot = makeTempRepo("cewp-integration-mcp-"); + try { + const responses = runMcp(repoRoot, [ + request(1, "initialize", { + protocolVersion: "2025-11-25", + capabilities: {}, + clientInfo: { name: "cewp-contract", version: "1.0.0" }, + }), + JSON.stringify({ jsonrpc: "2.0", method: "notifications/initialized" }), + request(2, "tools/list", {}), + call(3, "cewp_create", { + goal: "Update the bounded file", + scopes: ["README.md"], + verificationCommands: ["git diff --check"], + stoppingConditions: ["The diff check passes"], + }), + ]); + + assert(responses.length === 3, "notifications do not receive responses"); + assert(responses[0].result.protocolVersion === "2025-11-25", "supported protocol is negotiated"); + assert(responses[0].result.capabilities.tools, "server advertises tools capability"); + const names = responses[1].result.tools.map((tool) => tool.name); + assert(JSON.stringify(names) === JSON.stringify([ + "cewp_create", "cewp_inspect", "cewp_approve", "cewp_continue", + "cewp_retry", "cewp_revise", "cewp_verify", "cewp_finalize", + ]), "MCP exposes only the eight roadmap operations"); + const inspectTool = responses[1].result.tools.find((tool) => tool.name === "cewp_inspect"); + assert(inspectTool.annotations.readOnlyHint === false, "inspect truthfully declares its generated progress refresh"); + assert(responses[2].result.isError !== true, "create succeeds through MCP"); + const created = responses[2].result.structuredContent; + assert(created.run.status === "proposed", "create calls the supervised Core proposal service"); + assert(created.run.repo.root === repoRoot, "repository root is fixed to the MCP process cwd"); + + const runId = created.run.runId; + const gates = runMcp(repoRoot, [ + request(1, "initialize", { + protocolVersion: "2025-06-18", + capabilities: {}, + clientInfo: { name: "cewp-contract", version: "1.0.0" }, + }), + call(2, "cewp_inspect", { runId }), + call(3, "cewp_approve", { runId, confirm: false }), + call(4, "cewp_approve", { runId, confirm: true }), + call(5, "cewp_continue", { runId }), + call(6, "cewp_retry", { runId, confirm: false }), + call(7, "cewp_revise", { runId, goal: "Revised goal" }), + call(8, "cewp_verify", { runId }), + call(9, "cewp_finalize", { runId, confirm: false }), + call(10, "not_a_cewp_tool", {}), + call(11, "cewp_inspect", { runId: 7 }), + ]); + + assert(gates[1].result.structuredContent.run.status === "proposed", "inspect uses Core state inspection"); + assert(gates[2].result.isError === true && gates[2].result.content[0].text.includes("explicit confirmation"), "approve requires MCP confirmation"); + assert(gates[3].result.structuredContent.run.status === "approved", "confirmed approve reaches Core approval"); + assert(gates[4].result.isError === true && gates[4].result.content[0].text.includes("verified checkpoint"), "continue preserves checkpoint gate"); + assert(gates[5].result.isError === true && gates[5].result.content[0].text.includes("explicit confirmation"), "retry requires MCP confirmation before Core dispatch"); + assert(gates[6].result.structuredContent.run.status === "proposed", "revise calls Core and invalidates approval"); + assert(gates[7].result.isError === true && gates[7].result.content[0].text.includes("cannot verify"), "verify preserves Core state gate"); + assert(gates[8].result.isError === true && gates[8].result.content[0].text.includes("explicit confirmation"), "finalize requires MCP confirmation"); + assert(gates[9].error.code === -32602 && gates[9].error.message.includes("Unknown tool"), "unknown tools are protocol errors"); + assert(gates[10].error.code === -32602 && gates[10].error.message.includes("input schema"), "malformed tool arguments are protocol errors"); + } finally { + cleanupRepo(repoRoot); + } +} + +try { + runContract(); + console.log("[PASS] local MCP tools share supervised Core gates"); +} catch (error) { + console.error("[FAIL] local MCP integration contract"); + console.error(error && error.stack ? error.stack : error); + process.exitCode = 1; +} diff --git a/tests/contracts/plugin-package.js b/tests/contracts/plugin-package.js index b1f63e6..6520bdb 100644 --- a/tests/contracts/plugin-package.js +++ b/tests/contracts/plugin-package.js @@ -23,7 +23,14 @@ function runPluginPackageContract() { assert(manifest.version === packageJson.version, "plugin and npm versions stay aligned"); assert(manifest.skills === "./skills/", "plugin skill path is contained and relative"); assert(manifest.apps === undefined, "plugin does not claim an unbuilt app"); - assert(manifest.mcpServers === undefined, "plugin does not claim an unbuilt MCP server"); + assert(manifest.mcpServers === "./.mcp.json", "plugin declares one contained local MCP bundle"); + const mcpConfig = readJson(path.join(repoRoot, "plugins", "cewp", ".mcp.json")); + assert( + JSON.stringify(Object.keys(mcpConfig.mcpServers)) === JSON.stringify(["cewp"]), + "plugin MCP bundle declares only the CEWP local server", + ); + assert(mcpConfig.mcpServers.cewp.command === "cewp-mcp", "plugin MCP delegates to the installed CEWP Core binary"); + assert(packageJson.bin["cewp-mcp"] === "bin/cewp-mcp.js", "npm package exposes the declared MCP binary"); assert(manifest.hooks === "./hooks/hooks.json", "plugin declares one contained reviewable hook bundle"); const hookConfig = readJson(path.join(repoRoot, "plugins", "cewp", "hooks", "hooks.json")); assert( From 9cf7b91300f70fa61fe83ce77eac46ed30ee5a89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tuncay=20=C3=96lmez?= Date: Thu, 23 Jul 2026 09:30:00 +0300 Subject: [PATCH 12/40] Distinguish audit evidence from enforcement --- src/cli/parse.js | 5 +++ src/cli/usage.js | 1 + src/integration/binding.js | 78 ++++++++++++++++++++++++++++++++++++++ src/integration/cli.js | 19 ++++++++++ 4 files changed, 103 insertions(+) diff --git a/src/cli/parse.js b/src/cli/parse.js index e39fae8..730114d 100644 --- a/src/cli/parse.js +++ b/src/cli/parse.js @@ -142,6 +142,11 @@ function parseArgs(argv) { continue; } + if (args.command === "integration" && args.subcommand === "controls" && index === 2 && !arg.startsWith("--")) { + args.workflowRunId = arg; + continue; + } + if (args.command === "workflow" && args.subcommand === "validate" && index === 2 && !arg.startsWith("--")) { args.definitionFile = arg; continue; diff --git a/src/cli/usage.js b/src/cli/usage.js index 7a7aa4a..c4a5afa 100644 --- a/src/cli/usage.js +++ b/src/cli/usage.js @@ -19,6 +19,7 @@ Usage: cewp workflow finalize --yes [--json] cewp integration hooks approve --yes [--json] cewp integration hooks status [--json] + cewp integration controls [--json] cewp workflow revise --proposal [--from ] [--json] cewp workflow apply-revision --proposal --digest --yes [--json] cewp workflow migrate [--digest --yes] [--json] diff --git a/src/integration/binding.js b/src/integration/binding.js index b78b03e..7cbc46f 100644 --- a/src/integration/binding.js +++ b/src/integration/binding.js @@ -17,6 +17,7 @@ const { writeJsonAtomic } = require("../workflow/state"); const HOST_BINDING_SCHEMA_VERSION = "host-binding/v1"; const GENERATED_GOAL_BRIEF_SCHEMA_VERSION = "generated-goal-brief/v1"; +const INTEGRATION_CONTROL_RECEIPT_SCHEMA_VERSION = "integration-control-receipt/v1"; const HOST_SURFACES = Object.freeze([ "chatgpt-desktop", "codex-cli", @@ -209,6 +210,13 @@ function validateHostBinding(value, found) { const controls = Object.fromEntries( CONTROL_CLASSES.map((name) => [name, normalizeStringList(value.controls[name], `controls.${name}`)]), ); + const classifiedControls = CONTROL_CLASSES.flatMap((name) => controls[name]); + if (new Set(classifiedControls).size !== classifiedControls.length) { + throw new Error("Invalid host binding: a control cannot appear in more than one control class."); + } + if (execution.owner === "audit-only" && controls.preventive.length > 0) { + throw new Error("Invalid host binding: audit-only execution cannot claim preventive enforcement."); + } return { schemaVersion: HOST_BINDING_SCHEMA_VERSION, @@ -265,6 +273,58 @@ function bindingOwnershipPath(found) { return path.join(found.runRoot, "integration", "ownership.json"); } +function controlReceiptPath(found) { + return path.join(found.runRoot, "integration", "control-receipt.json"); +} + +function buildIntegrationControlReceipt(binding) { + const classifications = { + preventive: "preventive", + postExecution: "post-execution", + imported: "imported", + unavailable: "unavailable", + }; + const effects = { + preventive: "prevented-before-execution", + postExecution: "checked-after-execution", + imported: "observed-not-enforced", + unavailable: "unavailable", + }; + const controls = CONTROL_CLASSES.flatMap((classification) => ( + binding.controls[classification].map((name) => ({ + name, + classification: classifications[classification], + effect: effects[classification], + })) + )); + return { + schemaVersion: INTEGRATION_CONTROL_RECEIPT_SCHEMA_VERSION, + generatedAt: binding.provenance.recordedAt, + workflow: binding.workflow, + execution: binding.execution, + provenance: { + kind: binding.provenance.kind, + authenticationBoundary: binding.provenance.authenticationBoundary, + }, + controls, + summary: { + preventiveEnforced: binding.controls.preventive.length, + postExecutionChecked: binding.controls.postExecution.length, + importedObserved: binding.controls.imported.length, + unavailable: binding.controls.unavailable.length, + }, + claims: { + observedEvidenceIsPreventiveEnforcement: false, + providerExecutionSuppliesEnforcement: false, + preventiveControlAuthority: binding.controls.preventive.length > 0 ? "cewp-core" : "none", + guardrailAuthority: "cewp-core-outside-provider-execution", + }, + warnings: binding.execution.owner === "audit-only" + ? ["Audit-only evidence can be imported or checked after execution; it is not preventive enforcement."] + : [], + }; +} + function claimBindingWorktree(found, binding) { if (!binding.references.worktree) return null; if (!binding.workflow.taskId || !binding.workflow.checkpointId) { @@ -305,6 +365,7 @@ function createHostBinding(found, candidate, options = {}) { fs.mkdirSync(path.dirname(filePath), { recursive: true }); claimBindingWorktree(found, binding); writeJsonAtomic(filePath, binding); + writeJsonAtomic(controlReceiptPath(found), buildIntegrationControlReceipt(binding)); return binding; } @@ -314,6 +375,20 @@ function loadHostBinding(found) { return validateHostBinding(JSON.parse(fs.readFileSync(filePath, "utf8")), found); } +function loadIntegrationControlReceipt(found) { + const filePath = controlReceiptPath(found); + if (!fs.existsSync(filePath)) return null; + const receipt = JSON.parse(fs.readFileSync(filePath, "utf8")); + if (receipt.schemaVersion !== INTEGRATION_CONTROL_RECEIPT_SCHEMA_VERSION) { + throw new Error(`Invalid integration control receipt: expected ${INTEGRATION_CONTROL_RECEIPT_SCHEMA_VERSION}.`); + } + const binding = loadHostBinding(found); + if (!binding || JSON.stringify(receipt) !== JSON.stringify(buildIntegrationControlReceipt(binding))) { + throw new Error("Invalid integration control receipt: receipt does not match the validated host binding."); + } + return receipt; +} + function createGeneratedGoalBrief(found, taskId) { if (found.run.execution.owner !== "native") { throw new Error("Generated native goal briefs require native execution ownership."); @@ -354,8 +429,11 @@ module.exports = { GENERATED_GOAL_BRIEF_SCHEMA_VERSION, HOST_BINDING_SCHEMA_VERSION, HOST_SURFACES, + INTEGRATION_CONTROL_RECEIPT_SCHEMA_VERSION, + buildIntegrationControlReceipt, createGeneratedGoalBrief, createHostBinding, + loadIntegrationControlReceipt, loadHostBinding, validateHostBinding, }; diff --git a/src/integration/cli.js b/src/integration/cli.js index 26a10b9..c209b77 100644 --- a/src/integration/cli.js +++ b/src/integration/cli.js @@ -6,6 +6,8 @@ const { inspectCodexHookTrust, recordSubagentHookEvent, } = require("./hook-evidence"); +const { loadIntegrationControlReceipt } = require("./binding"); +const { loadWorkflowRun } = require("../workflow/state"); function outputJson(command, data) { console.log(JSON.stringify({ @@ -18,6 +20,23 @@ function outputJson(command, data) { } function runIntegration(options = {}) { + if (options.subcommand === "controls") { + if (!options.workflowRunId) throw new Error("integration controls requires a workflow run id."); + const found = loadWorkflowRun(process.cwd(), options.workflowRunId); + const result = loadIntegrationControlReceipt(found); + if (!result) throw new Error(`Integration control receipt not found for workflow run ${options.workflowRunId}.`); + if (options.json) outputJson("integration.controls", result); + else { + console.log("CEWP integration control receipt"); + console.log(`Run ID: ${result.workflow.runId}`); + console.log(`Owner: ${result.execution.owner}`); + console.log(`Preventive: ${result.summary.preventiveEnforced}`); + console.log(`Post-execution: ${result.summary.postExecutionChecked}`); + console.log(`Imported observations: ${result.summary.importedObserved}`); + console.log(`Unavailable: ${result.summary.unavailable}`); + } + return; + } if (options.subcommand === "hooks" && options.action === "ingest") { const raw = fs.readFileSync(0, "utf8"); if (Buffer.byteLength(raw, "utf8") > 64 * 1024) { From 39213b10075598a2c9118f06e1e0ca7b9eab2d1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tuncay=20=C3=96lmez?= Date: Thu, 23 Jul 2026 10:00:00 +0300 Subject: [PATCH 13/40] Document audit evidence boundaries --- README.md | 4 ++ docs/adr/0002-execution-ownership.md | 5 ++ docs/known-limitations.md | 1 + docs/workflow-runtime.md | 16 +++++ tests/contracts/integration-binding.js | 88 +++++++++++++++++++++++++- 5 files changed, 113 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 0864231..e76114a 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,10 @@ can expose `cewp_create`, `cewp_inspect`, `cewp_approve`, `cewp_continue`, `cewp `cewp_verify`, and `cewp_finalize` when the package-provided `cewp-mcp` command is on `PATH`. It does not gain direct access to the host's private thread, native goal lifecycle, billing data, or persistent UI. +For a workflow with a validated host binding, `cewp integration controls --json` shows +preventive, post-execution, imported-observation, and unavailable control classes without promoting +audit-only evidence into enforcement. + ## What CEWP Records Phase 9 supervised state lives under `.cewp/supervised-runs//`; graph workflow state lives under `.cewp/workflow-runs//`. Human-readable `progress.md` is generated from canonical state and cannot silently change it. diff --git a/docs/adr/0002-execution-ownership.md b/docs/adr/0002-execution-ownership.md index 3218f0d..84e2bdb 100644 --- a/docs/adr/0002-execution-ownership.md +++ b/docs/adr/0002-execution-ownership.md @@ -42,6 +42,11 @@ Receipts label each control as one of: Native and audit-only runs must not inherit managed claims. A hook or conversation warning may project a decision but never becomes the ownership or enforcement source. +The runtime materializes these classifications as `integration-control-receipt/v1` and exposes the receipt +through `cewp integration controls --json`. Audit-only bindings with preventive entries or +controls assigned to multiple classes are invalid. Imported entries explicitly render as observed, not +enforced, and receipt inspection verifies the artifact still matches its validated host binding. + ## Consequences The runtime needs a deterministic ownership registry and conflict fixtures before the supervised golden path ships. Recovery can safely resume only after worktree, plan, policy, owner, backend, and process state are compatible. App Server may later replace `codex-exec` for a managed checkpoint only through a new capability and migration decision; it cannot run beside it for the same checkpoint. diff --git a/docs/known-limitations.md b/docs/known-limitations.md index b03550f..abd8be4 100644 --- a/docs/known-limitations.md +++ b/docs/known-limitations.md @@ -11,6 +11,7 @@ CEWP is beta software. These limits are product boundaries, not hidden roadmap p - Numeric usage estimates stay unavailable until enough comparable local runs exist. When available, they are ranges with confidence and sample basis, never point promises. - File-level test-authoring enforcement recognizes common test directories and filename conventions. It cannot prove whether production code contains test-like logic. - The plugin contributes skills, a local stdio MCP bridge, and an optional review-required `SubagentStart`/`SubagentStop` evidence hook. The hook cannot expose a subagent thread id, does not read transcripts, and is never a Core enforcement boundary. MCP exposes only CEWP Core operations and does not attach to native host sessions. An Apps SDK card and App Server client are not shipped. +- Audit-only integration can validate imported evidence and record post-execution checks, but it cannot claim that CEWP prevented actions performed by the external owner. Its integration control receipt therefore permits no preventive entries. - Experimental OpenCode execution remains optional and outside the supervised golden path. Binary/version availability does not prove authentication or model readiness. - Manual is a non-executing handoff adapter. Claude, Gemini, Hermes, and other providers are not implemented. - Supervised worktree cleanup automation is not shipped; rollback is available for owned unverified work, and terminal evidence is retained for deliberate inspection/removal. diff --git a/docs/workflow-runtime.md b/docs/workflow-runtime.md index 1c07524..acaa7f9 100644 --- a/docs/workflow-runtime.md +++ b/docs/workflow-runtime.md @@ -36,6 +36,22 @@ A succeeded `task-result/v1` must include every approved baseline, targeted, and A failed `task-result/v1` is still evidence. CEWP validates its scope, approved verification commands, failure classification and signature, bounded output, and usage truth. It persists the result, accounts observed CEWP-controlled operations, and moves the checkpoint, task, and run to `blocked`. An identical canonical signature is derived as `repeated-failure`; a provider cannot self-declare it. Unknown host-internal work remains unknown and is not fabricated as observed usage. +## Integration Control Claims + +A validated host binding writes `integration/control-receipt.json` beside the provider-specific binding, +outside provider-neutral workflow state. Inspect it with: + +```bash +cewp integration controls --json +``` + +`integration-control-receipt/v1` classifies each named check as `preventive`, `postExecution`, `imported`, +or `unavailable`. One check cannot occupy multiple classes. Audit-only ownership cannot claim any preventive +control: imported evidence is rendered as `observed-not-enforced`, while a CEWP check performed after import +remains post-execution rather than preventive. The receipt is derived from and checked against the validated +host binding, so editing the artifact cannot promote an observation into enforcement. Guardrail authority +remains CEWP Core outside provider-controlled execution. + Recovery is explicit: retry, revise, reassign, a permitted pre-existing-failure waiver, rollback, cancel, or abandon. New regressions, repeated failures, scope gates, destructive-operation policy, and required reviewer PASS are non-waivable. A failed or unverified checkpoint never counts as completed. ## Budgets diff --git a/tests/contracts/integration-binding.js b/tests/contracts/integration-binding.js index 0fd7515..5b4d922 100644 --- a/tests/contracts/integration-binding.js +++ b/tests/contracts/integration-binding.js @@ -3,17 +3,20 @@ const fs = require("node:fs"); const path = require("node:path"); const { assert } = require("../harness/lib/assertions"); -const { cleanupRepo, makeTempRepo } = require("../harness/lib/temp-repo"); +const { cleanupRepo, makeTempRepo, runNode } = require("../harness/lib/temp-repo"); const { supportedSnapshot } = require("./integration-capabilities"); const { validDefinition } = require("./workflow-definition"); const { approveWorkflow } = require("./workflow-scheduler"); const { createGeneratedGoalBrief, createHostBinding, + loadIntegrationControlReceipt, loadHostBinding, } = require("../../src/integration/binding"); const { loadWorkflowRun, startWorkflowTask } = require("../../src/workflow/state"); +const cewpCli = path.join(__dirname, "..", "..", "bin", "cewp.js"); + function assertThrows(action, expected, label) { let error; try { @@ -36,6 +39,17 @@ function nativeDefinition() { return definition; } +function auditDefinition() { + const definition = validDefinition(); + definition.workflowId = "audit-integration"; + definition.execution = { + owner: "audit-only", + backend: null, + allowedModes: ["audit-only"], + }; + return definition; +} + function explicitBinding(runId) { return { schemaVersion: "host-binding/v1", @@ -129,6 +143,78 @@ function main() { cleanupRepo(managedRepo); } + const auditRepo = makeTempRepo("cewp-integration-audit-binding-"); + try { + const audit = approveWorkflow(auditRepo, auditDefinition()); + const auditFound = loadWorkflowRun(auditRepo, audit.runId); + const auditBinding = explicitBinding(audit.runId); + auditBinding.execution = { owner: "audit-only", backend: null }; + auditBinding.host.surface = "external-client"; + auditBinding.mode = "audit-import"; + auditBinding.provenance.kind = "imported-audit"; + auditBinding.references.goalId = null; + auditBinding.references.threadId = "external-thread-1"; + auditBinding.controls = { + preventive: ["scope-policy"], + postExecution: ["receipt-schema"], + imported: ["external-scope-observation"], + unavailable: ["provider-tool-prevention"], + }; + assertThrows( + () => createHostBinding(auditFound, auditBinding, { capabilities: supportedSnapshot() }), + /audit-only.*preventive/i, + "audit-only binding cannot claim preventive enforcement", + ); + + auditBinding.controls.preventive = []; + createHostBinding(auditFound, auditBinding, { capabilities: supportedSnapshot() }); + const controlReceipt = loadIntegrationControlReceipt(auditFound); + assert(controlReceipt.schemaVersion === "integration-control-receipt/v1", "control receipt is versioned"); + assert(controlReceipt.execution.owner === "audit-only", "receipt retains audit-only ownership"); + assert(controlReceipt.summary.preventiveEnforced === 0, "audit-only receipt claims no preventive enforcement"); + assert(controlReceipt.summary.postExecutionChecked === 1, "post-execution checks stay distinct"); + assert( + controlReceipt.controls.find((entry) => entry.name === "receipt-schema").classification === "post-execution", + "public receipt uses the documented post-execution classification", + ); + assert(controlReceipt.summary.importedObserved === 1, "imported observations stay distinct"); + assert( + controlReceipt.controls.find((entry) => entry.name === "external-scope-observation").effect === "observed-not-enforced", + "imported audit evidence is labeled observed rather than enforced", + ); + assert( + controlReceipt.claims.providerExecutionSuppliesEnforcement === false, + "audit receipt never treats provider-controlled execution as the enforcement source", + ); + const shown = runNode(cewpCli, ["integration", "controls", audit.runId, "--json"], auditRepo); + assert(shown.status === 0, `control receipt is available through operator JSON: ${shown.stderr}`); + const shownReceipt = JSON.parse(shown.stdout); + assert(shownReceipt.command === "integration.controls", "operator JSON identifies control inspection"); + assert(shownReceipt.data.summary.importedObserved === 1, "operator JSON preserves observed audit evidence"); + + const duplicate = { ...auditBinding, controls: { + ...auditBinding.controls, + imported: ["receipt-schema"], + } }; + assertThrows( + () => createHostBinding(auditFound, duplicate, { capabilities: supportedSnapshot(), replace: true }), + /more than one control class/, + "one control cannot receive conflicting enforcement classifications", + ); + + const receiptPath = path.join(auditFound.runRoot, "integration", "control-receipt.json"); + const tampered = JSON.parse(fs.readFileSync(receiptPath, "utf8")); + tampered.controls.find((entry) => entry.classification === "imported").effect = "prevented-before-execution"; + fs.writeFileSync(receiptPath, `${JSON.stringify(tampered, null, 2)}\n`); + assertThrows( + () => loadIntegrationControlReceipt(auditFound), + /does not match the validated host binding/, + "edited receipt cannot promote observed audit evidence to enforcement", + ); + } finally { + cleanupRepo(auditRepo); + } + const conflictRepo = makeTempRepo("cewp-integration-ownership-conflict-"); try { const native = approveWorkflow(conflictRepo, nativeDefinition()); From 945c1155a173e43807c61a4d834f1bbca91c9f1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tuncay=20=C3=96lmez?= Date: Thu, 23 Jul 2026 10:30:00 +0300 Subject: [PATCH 14/40] Document external Codex integration boundaries --- README.md | 1 + docs/codex-capability-matrix.md | 5 +-- docs/external-integration-boundary.md | 49 +++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 docs/external-integration-boundary.md diff --git a/README.md b/README.md index e76114a..43aa468 100644 --- a/README.md +++ b/README.md @@ -121,6 +121,7 @@ CEWP still ships ten reusable engineering skills and the earlier Coordinator Mod - [Install Guide](docs/install.md) - [Supervised Workflow](docs/supervised-workflow.md) +- [External Integration Boundary](docs/external-integration-boundary.md) - [Workflow Runtime](docs/workflow-runtime.md) - [Known Limitations](docs/known-limitations.md) - [Pilot Kit](docs/pilot-kit.md) diff --git a/docs/codex-capability-matrix.md b/docs/codex-capability-matrix.md index 3001d03..ca18edd 100644 --- a/docs/codex-capability-matrix.md +++ b/docs/codex-capability-matrix.md @@ -52,7 +52,7 @@ Schema presence does not prove that a plugin can attach to the desktop app's exi | `PreToolUse` deny output | supported | The deterministic fixture emits the documented `permissionDecision: deny` shape and is covered by `npm run test:hook-output`. | | `PreToolUse` as complete enforcement | unavailable | Official docs exclude or limit richer shell and non-MCP paths. A real CLI 0.137.0 Windows probe executed the requested PowerShell command despite the Bash deny hook. Core policy remains authoritative. | | Hook-based instant turn cancellation | unknown | Stop semantics do not establish instantaneous cancellation of an in-flight model or external process. | -| Local MCP to CEWP Core | unknown | MCP is supported by the host. Phase 11 implements a small Core-backed tool surface while conversation and CLI fallbacks remain required. | +| Local MCP to CEWP Core | supported | `cewp-mcp` implements the documented local stdio JSON-RPC lifecycle and exactly eight Core-backed tools. `npm run test:integration-mcp` proves schema validation, current-directory repository scope, Core state transitions, confirmation gates, business errors, and protocol errors without credentials. | ## App Server Boundary @@ -104,7 +104,7 @@ Reasons: - App Server adds useful goal metadata and lifecycle methods, but remains a separately owned experimental process with version drift and unresolved authenticated usage/cancellation behavior. - The spike did not demonstrate enough recovery or accounting advantage to justify shipping two incomplete managed backends. -The native fallback is a bounded generated goal brief plus supported host goal tools or explicit result intake. `audit-only` remains available for evidence supplied by another owner. The local MCP bridge is the next supported integration surface. Hooks and Apps SDK UI remain optional projections; their absence never weakens CEWP Core. See [ADR 0005](adr/0005-codex-integration-backend.md). +The native fallback is a bounded generated goal brief plus supported host goal tools or explicit result intake. `audit-only` remains available for evidence supplied by another owner. The local MCP bridge is the supported headless integration surface. Hooks and Apps SDK UI remain optional projections; their absence never weakens CEWP Core. See [ADR 0005](adr/0005-codex-integration-backend.md) and the [external integration boundary](external-integration-boundary.md). ## Reproduction @@ -118,6 +118,7 @@ npm run test:plugin-lifecycle npm run test:hook-output npm run test:integration-capabilities npm run test:integration-hook-evidence +npm run test:integration-mcp ``` The nested model probe is intentionally excluded from automated tests because it consumes account usage. Raw account values, credentials, thread ids, and machine-specific paths are not part of this document. diff --git a/docs/external-integration-boundary.md b/docs/external-integration-boundary.md new file mode 100644 index 0000000..90c4c94 --- /dev/null +++ b/docs/external-integration-boundary.md @@ -0,0 +1,49 @@ +# External Integration Boundary + +CEWP is a local workflow, governance, verification, and evidence runtime. A third-party interface may +present CEWP state in its own agent UI, but it must not become the execution owner merely by calling a +CEWP control surface. It remains responsible for its UI, user consent, authentication, transport lifecycle, +and any agent process it starts. + +## Supported Headless Surfaces + +`operator-json/v1` is the stable envelope shape for CLI inspection and control results. External tools may +invoke documented `cewp ... --json` commands, retain the `command`, `generatedAt`, `data`, and `warnings` +fields, and render them without converting warnings or observations into PASS. In particular, +`cewp integration controls --json` keeps preventive, post-execution, imported, and +unavailable controls distinct. + +`cewp-mcp` is the structured local stdio bridge. Configure its current working directory as exactly one +intended repository. It exposes create, inspect, approve, continue, retry, revise, verify, and finalize; the +tools import the same CEWP Core services as the CLI. An MCP client may add its own confirmation UI, but +cannot bypass Core approval, ownership, policy, effort, scope, budget, verification, receipt, or reviewer +gates. The server opens no network listener and provides no host account, billing, or private-session data. + +Hooks and conversation messages are optional projections. Hook evidence is separately trusted and +version-bound; a missing, disabled, stale, or malformed hook never changes Core enforcement. + +## Execution Ownership + +The external UI is not a fourth owner. Each run remains `managed`, `native`, or `audit-only`. A managed +checkpoint retains one backend and one CEWP-owned worktree. Native work remains host-owned. Audit-only +evidence may be imported or checked after execution but never presented as preventive enforcement. +Provider-specific host, goal, thread, turn, and subagent identifiers stay in integration sidecars rather +than provider-neutral workflow schemas. + +## Rich Codex Clients And App Server + +A client that needs rich Codex thread, turn, or goal lifecycle should integrate with the documented Codex +App Server and own that separate process, authentication boundary, thread identifiers, selected working +directory, interruption behavior, and cleanup. That client is separate from the CEWP plugin. Its process +does not attach to the ChatGPT desktop app's existing internal session, inherit private desktop credentials, +or turn App Server schema presence into plugin capability. + +CEWP has not graduated App Server as a managed backend. A request for that ungraduated backend falls back +to the selected `codex-exec` path, which already owns isolated dispatch, artifacts, verification, recovery, +and reviewer gates. External clients can still use CEWP MCP or operator JSON around their own UI without +changing this backend decision. + +CEWP will build no custom terminal-session protocol, terminal server, desktop shell, private Codex +protocol adapter, UI scraper, or undocumented desktop-session attachment. A richer client should compose +the supported Codex App Server and CEWP's headless surfaces rather than making CEWP a competing terminal +product. From 8134ecf9de6bb3e88f392ad220d1ea7eac040722 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tuncay=20=C3=96lmez?= Date: Thu, 23 Jul 2026 11:00:00 +0300 Subject: [PATCH 15/40] Test external integration capability claims --- package.json | 1 + tests/contracts/integration-capabilities.js | 17 +++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/package.json b/package.json index 1f1678b..c6145e0 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,7 @@ "docs/known-limitations.md", "docs/pilot-kit.md", "docs/codex-capability-matrix.md", + "docs/external-integration-boundary.md", "docs/adr/0001-codex-first-supervised-goals.md", "docs/adr/0002-execution-ownership.md", "docs/adr/0003-cost-assurance-and-safe-pauses.md", diff --git a/tests/contracts/integration-capabilities.js b/tests/contracts/integration-capabilities.js index 6679d4f..4eaa45c 100644 --- a/tests/contracts/integration-capabilities.js +++ b/tests/contracts/integration-capabilities.js @@ -131,6 +131,23 @@ function main() { assert(capabilityMatrix.includes("plugin install, disable, upgrade, and uninstall"), "plugin lifecycle evidence is current"); assert(!capabilityMatrix.includes("Phase 9 must test"), "capability matrix has no stale Phase 9 promise"); + const externalBoundary = fs.readFileSync( + path.join(repoRoot, "docs", "external-integration-boundary.md"), + "utf8", + ); + for (const required of [ + "operator-json/v1", + "cewp-mcp", + "current working directory", + "must not become the execution owner", + "Codex App Server", + "codex-exec", + "does not attach to the ChatGPT desktop app's existing internal session", + "no custom terminal-session protocol", + ]) { + assert(externalBoundary.includes(required), `external integration boundary documents ${required}`); + } + console.log("[PASS] Codex integration capability drift and backend decision stay truthful"); } From 371966b4046f154e34750bf579559191d307e51c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tuncay=20=C3=96lmez?= Date: Thu, 23 Jul 2026 11:30:00 +0300 Subject: [PATCH 16/40] Prepare Phase 11 beta release surface --- docs/codex-capability-matrix.md | 2 +- docs/external-integration-boundary.md | 2 ++ docs/release-notes.md | 28 +++++++++++++++ docs/supervised-workflow.md | 5 +++ package.json | 2 +- plugins/cewp/.codex-plugin/plugin.json | 2 +- src/mcp/server.js | 14 +++++++- tests/contracts/integration-hook-evidence.js | 2 +- tests/contracts/integration-mcp.js | 13 +++++++ tests/contracts/workflow-release.js | 36 +++++++++++--------- tests/harness/run-smoke.js | 2 +- 11 files changed, 85 insertions(+), 23 deletions(-) diff --git a/docs/codex-capability-matrix.md b/docs/codex-capability-matrix.md index ca18edd..769833a 100644 --- a/docs/codex-capability-matrix.md +++ b/docs/codex-capability-matrix.md @@ -52,7 +52,7 @@ Schema presence does not prove that a plugin can attach to the desktop app's exi | `PreToolUse` deny output | supported | The deterministic fixture emits the documented `permissionDecision: deny` shape and is covered by `npm run test:hook-output`. | | `PreToolUse` as complete enforcement | unavailable | Official docs exclude or limit richer shell and non-MCP paths. A real CLI 0.137.0 Windows probe executed the requested PowerShell command despite the Bash deny hook. Core policy remains authoritative. | | Hook-based instant turn cancellation | unknown | Stop semantics do not establish instantaneous cancellation of an in-flight model or external process. | -| Local MCP to CEWP Core | supported | `cewp-mcp` implements the documented local stdio JSON-RPC lifecycle and exactly eight Core-backed tools. `npm run test:integration-mcp` proves schema validation, current-directory repository scope, Core state transitions, confirmation gates, business errors, and protocol errors without credentials. | +| Local MCP to CEWP Core | supported | `cewp-mcp` implements the documented local stdio JSON-RPC lifecycle and exactly eight Core-backed tools. `npm run test:integration-mcp` proves schema validation, current-directory repository scope, Core state transitions, confirmation gates, business errors, protocol errors, and explicit protocol-version drift fallback without credentials. | ## App Server Boundary diff --git a/docs/external-integration-boundary.md b/docs/external-integration-boundary.md index 90c4c94..e1645eb 100644 --- a/docs/external-integration-boundary.md +++ b/docs/external-integration-boundary.md @@ -18,6 +18,8 @@ intended repository. It exposes create, inspect, approve, continue, retry, revis tools import the same CEWP Core services as the CLI. An MCP client may add its own confirmation UI, but cannot bypass Core approval, ownership, policy, effort, scope, budget, verification, receipt, or reviewer gates. The server opens no network listener and provides no host account, billing, or private-session data. +Unsupported MCP protocol versions return an explicit `mcp-protocol-version-drift` compatibility warning +and name `cewp-cli-operator-json` as the safe fallback. Hooks and conversation messages are optional projections. Hook evidence is separately trusted and version-bound; a missing, disabled, stale, or malformed hook never changes Core enforcement. diff --git a/docs/release-notes.md b/docs/release-notes.md index 20e756a..91ce82a 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -4,6 +4,34 @@ No changes yet. +## 0.11.0-beta.0 + +### Summary + +Codex-first native-goal supervision and integration bridge preparation. Phase 11 keeps managed, native, +and audit-only ownership separate, retains `codex-exec`, adds supported headless integration surfaces, +and preserves provider-neutral workflow state. This version is prepared locally and is not published, +tagged, or released; clean Linux validation remains required before the technical release gate can close. + +### Added + +- Enforced cross-mode worktree conflicts so native and managed ownership cannot target the same CEWP task worktree or active checkpoint. +- Added explicit implementation, repair, and reviewer task classes with operator-approved model/effort revisions and no automatic model routing. +- Added opt-in, exact-definition and version-bound `SubagentStart`/`SubagentStop` evidence. Hook absence, drift, malformed input, or host distrust leaves Core gates unchanged. +- Added a local stdio bridge with eight Core-backed MCP tools for create, inspect, approve, continue, retry, revise, verify, and finalize. MCP and CLI call the same services and preserve the same Core gates. +- Added MCP protocol-drift negotiation with a stable compatibility warning and CLI/operator-JSON fallback. +- Added structured host observations that keep observed, imported, stale, malformed, unavailable, and unknown truth states distinct without inventing billing impact. +- Added `integration-control-receipt/v1` and `cewp integration controls` so audit-only evidence cannot be presented as preventive enforcement. +- Added a packaged external-integration boundary for third-party MCP/operator JSON clients and rich Codex clients that own a separate App Server lifecycle. + +### Changed + +- App Server remains ungraduated because no material supported lifecycle, usage, or recovery advantage was proven. An explicit request retains the `codex-exec` fallback. +- Provider-specific host, goal, thread, turn, subagent, and worktree references stay outside provider-neutral workflow schemas. +- Host goal completion, hook completion, and imported evidence never count as CEWP verification or independent reviewer PASS. +- Independent external pilot evidence remains Phase 13 validation debt; fixtures, maintainer dogfood, and multiple machines used by one maintainer do not satisfy it. +- No provider, desktop UI, terminal server, merge, push, publish, tag, or release automation was added. + ## 0.10.0-beta.0 ### Summary diff --git a/docs/supervised-workflow.md b/docs/supervised-workflow.md index 86a25f1..c718924 100644 --- a/docs/supervised-workflow.md +++ b/docs/supervised-workflow.md @@ -170,6 +170,11 @@ and independent-review checks remain in Core. MCP host approval is an additional never substitutes for those checks. Business-rule failures return structured tool errors; malformed calls and unknown tools return JSON-RPC protocol errors. +During initialization, a supported MCP protocol version is echoed with `compatibility.compatible: true`. +An unsupported version negotiates the current supported version but also returns the stable +`mcp-protocol-version-drift` warning and `cewp-cli-operator-json` fallback; clients should disconnect or +use that fallback rather than assuming newer semantics. + The plugin MCP entry expects the package-provided `cewp-mcp` binary to be on `PATH`. If a third-party MCP client cannot discover the plugin manifest, configure a local stdio server with command `cewp-mcp` and set its working directory to the intended repository. Do not point one server process at multiple repositories. diff --git a/package.json b/package.json index c6145e0..c2fba85 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@setrathex/codex-engineering-workflow-pack", - "version": "0.10.0-beta.0", + "version": "0.11.0-beta.0", "description": "Long-running Codex goals without blind runs: local-first supervision, verification, recovery, review, and evidence.", "license": "MIT", "repository": { diff --git a/plugins/cewp/.codex-plugin/plugin.json b/plugins/cewp/.codex-plugin/plugin.json index e9c6ea0..2c21b5c 100644 --- a/plugins/cewp/.codex-plugin/plugin.json +++ b/plugins/cewp/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "cewp", - "version": "0.10.0-beta.0", + "version": "0.11.0-beta.0", "description": "Plan, run, recover, and review bounded engineering checkpoints through the local CEWP runtime.", "author": { "name": "SetraTheXX", diff --git a/src/mcp/server.js b/src/mcp/server.js index b83de69..aeb9f2a 100644 --- a/src/mcp/server.js +++ b/src/mcp/server.js @@ -49,7 +49,8 @@ function createMcpSession(options = {}) { } if (message.method === "initialize") { const requested = message.params && message.params.protocolVersion; - const protocolVersion = PROTOCOL_VERSIONS.has(requested) ? requested : "2025-11-25"; + const compatible = PROTOCOL_VERSIONS.has(requested); + const protocolVersion = compatible ? requested : "2025-11-25"; initialized = true; return response(message.id, { protocolVersion, @@ -60,6 +61,17 @@ function createMcpSession(options = {}) { description: "Local CEWP Core tools with supervised approval and evidence gates.", }, instructions: "Inspect before mutating. Explicit confirmation does not bypass CEWP Core state, ownership, scope, policy, budget, verification, or reviewer gates.", + compatibility: compatible + ? { compatible: true, requestedProtocolVersion: requested, fallback: null, warning: null } + : { + compatible: false, + requestedProtocolVersion: requested || null, + fallback: "cewp-cli-operator-json", + warning: { + code: "mcp-protocol-version-drift", + message: `Requested MCP protocol ${requested || "unknown"} is unsupported; negotiated ${protocolVersion}. Disconnect or use the CEWP CLI operator JSON fallback.`, + }, + }, }); } if (!initialized) return rpcError(message.id, -32002, "MCP session is not initialized."); diff --git a/tests/contracts/integration-hook-evidence.js b/tests/contracts/integration-hook-evidence.js index 72c05d6..3422e86 100644 --- a/tests/contracts/integration-hook-evidence.js +++ b/tests/contracts/integration-hook-evidence.js @@ -44,7 +44,7 @@ function main() { const output = JSON.parse(approved.stdout); assert(output.command === "integration.hooks.approve", "approval identifies the public command"); assert(output.data.trust.schemaVersion === "codex-hook-trust/v1", "hook trust is versioned"); - assert(output.data.trust.cewpVersion === "0.10.0-beta.0", "approval binds the CEWP runtime version"); + assert(output.data.trust.cewpVersion === "0.11.0-beta.0", "approval binds the CEWP runtime version"); assert(output.data.trust.codexVersion === "codex-cli 0.200.0", "approval binds the observed Codex version"); assert(/^sha256:[a-f0-9]{64}$/.test(output.data.trust.bundleDigest), "approval binds the exact hook bundle"); assert(output.data.nextAction.command === "/hooks", "approval still requires the host trust review"); diff --git a/tests/contracts/integration-mcp.js b/tests/contracts/integration-mcp.js index a0b0963..2b962ab 100644 --- a/tests/contracts/integration-mcp.js +++ b/tests/contracts/integration-mcp.js @@ -46,6 +46,7 @@ function runContract() { assert(responses.length === 3, "notifications do not receive responses"); assert(responses[0].result.protocolVersion === "2025-11-25", "supported protocol is negotiated"); + assert(responses[0].result.compatibility.compatible === true, "matching MCP protocol is compatible"); assert(responses[0].result.capabilities.tools, "server advertises tools capability"); const names = responses[1].result.tools.map((tool) => tool.name); assert(JSON.stringify(names) === JSON.stringify([ @@ -88,6 +89,18 @@ function runContract() { assert(gates[8].result.isError === true && gates[8].result.content[0].text.includes("explicit confirmation"), "finalize requires MCP confirmation"); assert(gates[9].error.code === -32602 && gates[9].error.message.includes("Unknown tool"), "unknown tools are protocol errors"); assert(gates[10].error.code === -32602 && gates[10].error.message.includes("input schema"), "malformed tool arguments are protocol errors"); + + const drift = runMcp(repoRoot, [ + request(1, "initialize", { + protocolVersion: "2099-01-01", + capabilities: {}, + clientInfo: { name: "future-client", version: "1.0.0" }, + }), + ])[0].result; + assert(drift.protocolVersion === "2025-11-25", "unsupported MCP version negotiates a supported version"); + assert(drift.compatibility.compatible === false, "MCP protocol drift is explicit"); + assert(drift.compatibility.warning.code === "mcp-protocol-version-drift", "MCP drift has a stable warning code"); + assert(drift.compatibility.fallback === "cewp-cli-operator-json", "MCP drift names the CLI fallback"); } finally { cleanupRepo(repoRoot); } diff --git a/tests/contracts/workflow-release.js b/tests/contracts/workflow-release.js index 23e0064..839b8d1 100644 --- a/tests/contracts/workflow-release.js +++ b/tests/contracts/workflow-release.js @@ -13,36 +13,38 @@ const plugin = JSON.parse(fs.readFileSync( const releaseNotes = fs.readFileSync(path.join(repoRoot, "docs", "release-notes.md"), "utf8"); function runWorkflowReleaseContract() { - assert(packageJson.version === "0.10.0-beta.0", "Phase 10 package version is exact"); + assert(packageJson.version === "0.11.0-beta.0", "Phase 11 package version is exact"); assert(plugin.version === packageJson.version, "plugin version follows the package version"); const unreleasedIndex = releaseNotes.indexOf("## Unreleased"); - const releaseIndex = releaseNotes.indexOf("## 0.10.0-beta.0"); - const previousIndex = releaseNotes.indexOf("## 0.8.0-beta.0"); - assert(unreleasedIndex >= 0 && releaseIndex > unreleasedIndex, "fresh Unreleased precedes the Phase 10 release"); - assert(previousIndex > releaseIndex, "Phase 10 release precedes earlier release history"); + const releaseIndex = releaseNotes.indexOf("## 0.11.0-beta.0"); + const previousIndex = releaseNotes.indexOf("## 0.10.0-beta.0"); + assert(unreleasedIndex >= 0 && releaseIndex > unreleasedIndex, "fresh Unreleased precedes the Phase 11 release"); + assert(previousIndex > releaseIndex, "Phase 11 release precedes earlier release history"); const unreleased = releaseNotes.slice(unreleasedIndex, releaseIndex); assert(unreleased.includes("No changes yet."), "fresh Unreleased is explicitly empty"); for (const claim of [ - "supervised golden path", - "workflow-compiler-request/v1", - "workflow-definition/v1", - "run-state/v2", - "one-, two-, and four-worker", - "OpenCode remains experimental", - "independent external pilot evidence remains Phase 13 validation debt", - "No provider, desktop UI, terminal server, native-goal control, merge, push, publish, tag, or release automation", + "native and managed ownership", + "no automatic model routing", + "SubagentStart", + "eight Core-backed MCP tools", + "observed, imported, stale, malformed, unavailable, and unknown", + "audit-only", + "App Server remains ungraduated", + "`codex-exec` fallback", + "external pilot evidence remains Phase 13 validation debt", + "No provider, desktop UI, terminal server, merge, push, publish, tag, or release automation", ]) { - assert(releaseNotes.slice(releaseIndex, previousIndex).includes(claim), `Phase 10 notes include honest claim: ${claim}`); + assert(releaseNotes.slice(releaseIndex, previousIndex).includes(claim), `Phase 11 notes include honest claim: ${claim}`); } - assert(packageJson.files.includes("docs/workflow-runtime.md"), "Phase 10 runtime guide is in the package surface"); + assert(packageJson.files.includes("docs/external-integration-boundary.md"), "Phase 11 boundary guide is in the package surface"); } try { runWorkflowReleaseContract(); - console.log("[PASS] Phase 10 version and release surface are aligned"); + console.log("[PASS] Phase 11 version and release surface are aligned"); } catch (error) { - console.error("[FAIL] Phase 10 release surface contract"); + console.error("[FAIL] Phase 11 release surface contract"); console.error(error && error.stack ? error.stack : error); process.exitCode = 1; } diff --git a/tests/harness/run-smoke.js b/tests/harness/run-smoke.js index c176348..c442adb 100644 --- a/tests/harness/run-smoke.js +++ b/tests/harness/run-smoke.js @@ -2572,7 +2572,7 @@ async function main() { const pack = run("npm", ["pack", "--dry-run"], { cwd: cewpRoot, timeout: 120000 }); const packOutput = `${pack.stdout}\n${pack.stderr}`; assertExit(pack, 0, "npm pack --dry-run"); - assert(packageJson.version === "0.10.0-beta.0", `unexpected package version: ${packageJson.version}`); + assert(packageJson.version === "0.11.0-beta.0", `unexpected package version: ${packageJson.version}`); assert(packOutput.includes("docs/adapter-contract.md"), "adapter contract doc should be packed"); assert(packOutput.includes("docs/supervised-workflow.md"), "supervised workflow doc should be packed"); assert(packOutput.includes("docs/known-limitations.md"), "known limitations should be packed"); From 59d2039f7ec347ecc489cbfea2b175e87ce09c84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tuncay=20=C3=96lmez?= Date: Thu, 23 Jul 2026 12:00:00 +0300 Subject: [PATCH 17/40] Add normalized evidence receipts --- README.md | 1 + docs/evidence-receipts.md | 39 +++ docs/release-notes.md | 5 +- package.json | 8 +- src/cli/parse.js | 2 +- src/cli/usage.js | 1 + src/evidence/receipt.js | 368 ++++++++++++++++++++++++++++ src/workflow/cli.js | 15 ++ src/workflow/state.js | 4 + tests/contracts/evidence-receipt.js | 148 +++++++++++ tests/contracts/workflow-release.js | 2 +- 11 files changed, 588 insertions(+), 5 deletions(-) create mode 100644 docs/evidence-receipts.md create mode 100644 src/evidence/receipt.js create mode 100644 tests/contracts/evidence-receipt.js diff --git a/README.md b/README.md index 43aa468..900023e 100644 --- a/README.md +++ b/README.md @@ -122,6 +122,7 @@ CEWP still ships ten reusable engineering skills and the earlier Coordinator Mod - [Install Guide](docs/install.md) - [Supervised Workflow](docs/supervised-workflow.md) - [External Integration Boundary](docs/external-integration-boundary.md) +- [Evidence Receipts](docs/evidence-receipts.md) - [Workflow Runtime](docs/workflow-runtime.md) - [Known Limitations](docs/known-limitations.md) - [Pilot Kit](docs/pilot-kit.md) diff --git a/docs/evidence-receipts.md b/docs/evidence-receipts.md new file mode 100644 index 0000000..32addbd --- /dev/null +++ b/docs/evidence-receipts.md @@ -0,0 +1,39 @@ +# Evidence Receipts + +`cewp workflow receipt ` writes `evidence-receipt.json` and `evidence-receipt.md` under the local +workflow run directory. `--json` wraps the same receipt and paths in `operator-json/v1` for external tools. +Receipt generation does not execute an agent, verification command, or control process. + +`evidence-receipt/v1` is a normalized read model over the approved workflow definition, canonical run +state, checkpoints, validated task results, interventions, events, independent reviews, integration control +receipt, and referenced evidence files. It includes: + +- goal and source-plan identity; +- workflow digest and revision history; +- operating modes, execution owner, and backend; +- tasks, attempts, changed files, scope verdicts, commands, and verification evidence; +- failure and recovery fields, interventions, review decision, and timestamps; +- observed usage where supported and explicit unknown values otherwise; +- the full approved and consumed budget envelope; +- git base/head identities for new runs; +- warnings and a sorted integrity inventory. + +The JSON model is deterministic for unchanged inputs when `generatedAt` is fixed. The CLI timestamp is the +documented variable. Historical runs that predate a field retain an explicit unknown. Runs that are not +finalized, have malformed event/evidence data, or are missing referenced evidence produce a partial receipt +with warnings rather than a complete claim. + +## Integrity Boundary + +Integrity entries contain byte length and `sha256` for canonical run evidence, the approved definition, +and referenced evidence files. Source identities include workflow/source hashes when available plus git +base and receipt-time head commits. This is `tamper-evident-local-metadata`, not tamper-proof storage: an +attacker who can rewrite both evidence and metadata can replace both. Durable signing or remote attestation +is not implied. + +## Privacy Boundary + +Raw prompts, adapter output, transcripts, and raw log contents are excluded by default. The receipt may +contain repository-relative paths, changed filenames, approved commands, bounded failure/review summaries, +and artifact names. Inspect these fields before sharing. Phase 12 privacy work will add export redaction; +until then, keep the receipt local if those metadata fields are sensitive. diff --git a/docs/release-notes.md b/docs/release-notes.md index 91ce82a..3507a7b 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -2,7 +2,10 @@ ## Unreleased -No changes yet. +### Phase 12 development + +- Added deterministic `evidence-receipt/v1` JSON/Markdown generation for workflow runs with complete-versus-partial truth, task/checkpoint/review evidence, usage unknowns, budget compliance, git identities, and local SHA-256 integrity metadata. +- Added `cewp workflow receipt ` without executing agents or verification commands. Raw prompts, transcripts, adapter output, and raw log contents remain excluded by default. ## 0.11.0-beta.0 diff --git a/package.json b/package.json index c2fba85..b6c0055 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "docs/pilot-kit.md", "docs/codex-capability-matrix.md", "docs/external-integration-boundary.md", + "docs/evidence-receipts.md", "docs/adr/0001-codex-first-supervised-goals.md", "docs/adr/0002-execution-ownership.md", "docs/adr/0003-cost-assurance-and-safe-pauses.md", @@ -51,7 +52,8 @@ "node": ">=22" }, "scripts": { - "test": "npm run test:contracts && npm run smoke", + "test": "npm run test:contracts && npm run test:phase12 && npm run smoke", + "test:phase12": "npm run test:evidence-receipt", "test:contracts": "npm run test:init-install && npm run test:adapter-profile && npm run test:doctor-json && npm run test:operator-json && npm run test:hook-output && npm run test:skill-format && npm run test:ownership-gates && npm run test:fixtures && npm run test:plugin-package && npm run test:supervised-intake && npm run test:supervised-proposal && npm run test:supervised-controls && npm run test:supervised-execution && npm run test:supervised-review && npm run test:supervised-failure && npm run test:supervised-linear-resume && npm run test:supervised-demo && npm run test:workflow-definition && npm run test:workflow-compiler && npm run test:workflow-proposal && npm run test:workflow-scheduler && npm run test:workflow-worker-matrix && npm run test:workflow-result && npm run test:workflow-failure-result && npm run test:workflow-state-machine && npm run test:workflow-interventions && npm run test:workflow-failure-matrix && npm run test:workflow-budget && npm run test:workflow-progress && npm run test:workflow-review && npm run test:workflow-checkpoint-review && npm run test:workflow-lifecycle && npm run test:workflow-revision && npm run test:workflow-migration && npm run test:workflow-templates && npm run test:workflow-docs && npm run test:workflow-release && npm run test:integration-capabilities && npm run test:integration-binding && npm run test:integration-effort-policy && npm run test:integration-hook-evidence && npm run test:integration-mcp && npm run test:integration-observation && npm run test:native-goal-events", "test:init-install": "node tests/contracts/init-install.js", "test:adapter-profile": "node tests/contracts/adapter-profile.js", @@ -95,6 +97,7 @@ "test:integration-effort-policy": "node tests/contracts/integration-effort-policy.js", "test:integration-hook-evidence": "node tests/contracts/integration-hook-evidence.js", "test:integration-mcp": "node tests/contracts/integration-mcp.js", + "test:evidence-receipt": "node tests/contracts/evidence-receipt.js", "test:integration-observation": "node tests/contracts/integration-observation.js", "test:native-goal-events": "node tests/contracts/native-goal-events.js", "demo:supervised": "node src/demo/supervised.js", @@ -106,7 +109,8 @@ "check:syntax": "node --check ./bin/cewp.js && node --check ./src/demo/fake-codex.js && node --check ./src/demo/supervised.js && node --check ./src/integration/capabilities.js && node --check ./src/integration/binding.js && node --check ./src/integration/cli.js && node --check ./src/integration/effort-policy.js && node --check ./src/integration/hook-evidence.js && node --check ./src/integration/observation.js && node --check ./src/integration/native-goal.js && node --check ./src/skills/install.js && node --check ./src/skills/format.js && node --check ./src/skills/doctor-report.js && node --check ./src/run/ownership.js && node --check ./src/run/control-gates.js && node --check ./src/supervise/budget.js && node --check ./src/supervise/cli.js && node --check ./src/supervise/commands.js && node --check ./src/supervise/controls.js && node --check ./src/supervise/execution.js && node --check ./src/supervise/profiles.js && node --check ./src/supervise/receipt.js && node --check ./src/supervise/review.js && node --check ./src/supervise/state.js && node --check ./src/supervise/test-authoring.js && node --check ./src/supervise/verification.js && node --check ./src/workflow/cli.js && node --check ./src/workflow/definition.js && node --check ./src/workflow/graph.js && node --check ./src/workflow/result.js && node --check ./src/workflow/scheduler.js && node --check ./src/workflow/source.js && node --check ./src/workflow/state.js && node --check ./src/workflow/transitions.js && node --check ./plugins/cewp/hooks/capture-subagent.js && node --check ./tests/contracts/init-install.js && node --check ./tests/contracts/adapter-profile.js && node --check ./tests/contracts/doctor-json.js && node --check ./tests/contracts/operator-json.js && node --check ./tests/contracts/hook-output.js && node --check ./tests/contracts/skill-format.js && node --check ./tests/contracts/ownership-gates.js && node --check ./tests/contracts/deterministic-fixtures.js && node --check ./tests/contracts/plugin-package.js && node --check ./tests/contracts/supervised-intake.js && node --check ./tests/contracts/supervised-proposal.js && node --check ./tests/contracts/supervised-controls.js && node --check ./tests/contracts/supervised-execution.js && node --check ./tests/contracts/supervised-review.js && node --check ./tests/contracts/supervised-failure.js && node --check ./tests/contracts/supervised-linear-resume.js && node --check ./tests/contracts/supervised-demo.js && node --check ./tests/contracts/integration-capabilities.js && node --check ./tests/contracts/integration-binding.js && node --check ./tests/contracts/integration-effort-policy.js && node --check ./tests/contracts/integration-hook-evidence.js && node --check ./tests/contracts/integration-observation.js && node --check ./tests/contracts/native-goal-events.js && node --check ./tests/contracts/workflow-definition.js && node --check ./tests/contracts/workflow-proposal.js && node --check ./tests/contracts/workflow-scheduler.js && node --check ./tests/contracts/workflow-result.js && node --check ./tests/contracts/workflow-state-machine.js && node --check ./tests/contracts/workflow-interventions.js && node --check ./tests/capabilities/clean-install.js && node --check ./tests/capabilities/plugin-lifecycle.js && node --check ./tests/capabilities/codex-app-server.js && node --check ./tests/capabilities/fixtures/deny-shell-hook.js", "check:workflow-syntax": "node --check ./src/workflow/budget.js && node --check ./src/workflow/compiler.js && node --check ./src/workflow/migration.js && node --check ./src/workflow/progress.js && node --check ./src/workflow/proposal.js && node --check ./src/workflow/review.js && node --check ./src/workflow/revision.js && node --check ./src/workflow/templates.js && node --check ./tests/contracts/workflow-budget.js && node --check ./tests/contracts/workflow-checkpoint-review.js && node --check ./tests/contracts/workflow-compiler.js && node --check ./tests/contracts/workflow-docs.js && node --check ./tests/contracts/workflow-failure-matrix.js && node --check ./tests/contracts/workflow-failure-result.js && node --check ./tests/contracts/workflow-migration.js && node --check ./tests/contracts/workflow-progress.js && node --check ./tests/contracts/workflow-release.js && node --check ./tests/contracts/workflow-review.js && node --check ./tests/contracts/workflow-lifecycle.js && node --check ./tests/contracts/workflow-revision.js && node --check ./tests/contracts/workflow-templates.js && node --check ./tests/contracts/workflow-worker-matrix.js", "check:mcp": "node --check ./bin/cewp-mcp.js && node --check ./src/mcp/server.js && node --check ./src/mcp/tools.js && node --check ./tests/contracts/integration-mcp.js", - "check": "npm run check:syntax && npm run check:workflow-syntax && npm run check:mcp && npm test", + "check:evidence": "node --check ./src/evidence/receipt.js && node --check ./tests/contracts/evidence-receipt.js", + "check": "npm run check:syntax && npm run check:workflow-syntax && npm run check:mcp && npm run check:evidence && npm test", "pack:dry-run": "npm pack --dry-run" }, "keywords": [ diff --git a/src/cli/parse.js b/src/cli/parse.js index 730114d..b76bf5f 100644 --- a/src/cli/parse.js +++ b/src/cli/parse.js @@ -157,7 +157,7 @@ function parseArgs(argv) { continue; } - if (args.command === "workflow" && ["status", "start", "result", "review", "finalize", "intervene", "revise", "apply-revision", "migrate"].includes(args.subcommand) && index === 2 && !arg.startsWith("--")) { + if (args.command === "workflow" && ["status", "start", "result", "review", "finalize", "receipt", "intervene", "revise", "apply-revision", "migrate"].includes(args.subcommand) && index === 2 && !arg.startsWith("--")) { args.workflowRunId = arg; continue; } diff --git a/src/cli/usage.js b/src/cli/usage.js index c4a5afa..2283f04 100644 --- a/src/cli/usage.js +++ b/src/cli/usage.js @@ -17,6 +17,7 @@ Usage: cewp workflow result --task --result --yes [--json] cewp workflow review --result --yes [--json] cewp workflow finalize --yes [--json] + cewp workflow receipt [--json] cewp integration hooks approve --yes [--json] cewp integration hooks status [--json] cewp integration controls [--json] diff --git a/src/evidence/receipt.js b/src/evidence/receipt.js new file mode 100644 index 0000000..c33b872 --- /dev/null +++ b/src/evidence/receipt.js @@ -0,0 +1,368 @@ +"use strict"; + +const crypto = require("node:crypto"); +const fs = require("node:fs"); +const path = require("node:path"); +const { getGitHeadCommit } = require("../lib/git"); +const { normalizeSlashPath } = require("../lib/paths"); +const { loadIntegrationControlReceipt } = require("../integration/binding"); +const { writeJsonAtomic } = require("../workflow/state"); + +const EVIDENCE_RECEIPT_SCHEMA_VERSION = "evidence-receipt/v1"; +const RECEIPT_BASENAMES = new Set(["evidence-receipt.json", "evidence-receipt.md"]); + +function lexicalCompare(left, right) { + return left < right ? -1 : left > right ? 1 : 0; +} + +function isInside(parentPath, childPath) { + const relative = path.relative(path.resolve(parentPath), path.resolve(childPath)); + return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative)); +} + +function readJson(filePath, label) { + try { + return JSON.parse(fs.readFileSync(filePath, "utf8")); + } catch (error) { + throw new Error(`Invalid ${label}: ${error.message}`); + } +} + +function listFiles(rootPath) { + if (!fs.existsSync(rootPath)) return []; + const files = []; + const visit = (current) => { + for (const entry of fs.readdirSync(current, { withFileTypes: true })) { + const entryPath = path.join(current, entry.name); + if (entry.isDirectory()) visit(entryPath); + else if (entry.isFile()) files.push(entryPath); + } + }; + visit(rootPath); + return files.sort((left, right) => lexicalCompare(normalizeSlashPath(left), normalizeSlashPath(right))); +} + +function readJsonDirectory(found, rootPath, label, warnings) { + return listFiles(rootPath) + .filter((filePath) => filePath.endsWith(".json")) + .map((filePath) => { + try { + return { path: filePath, value: readJson(filePath, label) }; + } catch (error) { + warnings.push({ + code: "malformed-evidence-file", + path: normalizeSlashPath(path.relative(found.repoRoot, filePath)), + message: error.message, + }); + return null; + } + }) + .filter(Boolean); +} + +function readEvents(found, warnings) { + const eventPath = path.join(found.runRoot, "events.jsonl"); + if (!fs.existsSync(eventPath)) { + warnings.push({ code: "missing-events", message: "Workflow event ledger is missing." }); + return []; + } + return fs.readFileSync(eventPath, "utf8").split(/\r?\n/).filter(Boolean).map((line, index) => { + try { + return JSON.parse(line); + } catch { + warnings.push({ code: "malformed-event", line: index + 1, message: "Workflow event is not valid JSON." }); + return { schemaVersion: "event/v1", type: "malformed", line: index + 1 }; + } + }); +} + +function truthAggregate(values, reason) { + const present = values.filter(Boolean); + if (present.length > 0 && present.every((entry) => entry.label === "observed")) { + return { + label: "observed", + value: present.reduce((total, entry) => total + entry.value, 0), + sources: [...new Set(present.map((entry) => entry.source).filter(Boolean))].sort(), + }; + } + return { label: "unknown", value: null, reason }; +} + +function safeEvidencePath(found, relativePath) { + if (typeof relativePath !== "string" || relativePath.length === 0) return null; + const resolved = path.resolve(found.repoRoot, relativePath); + return isInside(found.repoRoot, resolved) ? resolved : null; +} + +function referencedPaths(results, reviews) { + const paths = []; + for (const result of results) { + for (const entry of result.value.verification.baseline.evidence || []) paths.push(entry.evidencePath); + for (const entry of result.value.verification.targeted || []) paths.push(entry.evidencePath); + for (const entry of result.value.verification.full || []) paths.push(entry.evidencePath); + for (const artifact of result.value.artifacts || []) paths.push(artifact.path); + for (const evidencePath of (result.value.failure && result.value.failure.evidencePaths) || []) paths.push(evidencePath); + } + for (const review of reviews) { + for (const evidence of review.value.evidence || []) paths.push(evidence.path); + for (const finding of review.value.findings || []) { + for (const evidencePath of finding.evidencePaths || []) paths.push(evidencePath); + } + } + return [...new Set(paths)].sort(); +} + +function hashFile(found, filePath) { + const content = fs.readFileSync(filePath); + return { + path: normalizeSlashPath(path.relative(found.repoRoot, filePath)), + sha256: `sha256:${crypto.createHash("sha256").update(content).digest("hex")}`, + bytes: content.length, + }; +} + +function integrityInventory(found, results, reviews, warnings) { + const canonical = listFiles(found.runRoot).filter((filePath) => { + if (RECEIPT_BASENAMES.has(path.basename(filePath))) return false; + const relative = normalizeSlashPath(path.relative(found.runRoot, filePath)); + return relative === "run.json" + || relative === "events.jsonl" + || relative.startsWith("checkpoints/") + || relative.startsWith("results/") + || relative.startsWith("reviews/") + || relative.startsWith("integration/"); + }); + const definitionPath = path.resolve(found.repoRoot, found.run.workflow.definitionPath); + if (isInside(found.repoRoot, definitionPath) && fs.existsSync(definitionPath)) canonical.push(definitionPath); + for (const relativePath of referencedPaths(results, reviews)) { + const filePath = safeEvidencePath(found, relativePath); + if (!filePath || !fs.existsSync(filePath) || !fs.statSync(filePath).isFile()) { + warnings.push({ code: "missing-referenced-evidence", path: relativePath, message: "Referenced evidence file is unavailable." }); + continue; + } + canonical.push(filePath); + } + return [...new Set(canonical.map((entry) => path.resolve(entry)))] + .map((filePath) => hashFile(found, filePath)) + .sort((left, right) => lexicalCompare(left.path, right.path)); +} + +function taskReceipt(found, runtimeTask, resultsByTask) { + const definitionTask = found.definition.tasks.find((entry) => entry.id === runtimeTask.id); + const result = resultsByTask.get(runtimeTask.id) || null; + return { + id: runtimeTask.id, + title: definitionTask.title, + status: runtimeTask.status, + attempts: runtimeTask.attempts, + allowedFiles: definitionTask.allowedFiles, + forbiddenFiles: definitionTask.forbiddenFiles, + stoppingConditions: definitionTask.stoppingConditions, + changedFiles: result ? result.changedFiles : [], + scopeVerdict: result ? { status: "passed", basis: "validated-task-result" } : { status: "unknown", basis: "result-unavailable" }, + verification: result ? result.verification : { + baseline: { status: "unknown", evidence: [] }, + targeted: [], + full: [], + }, + failure: result ? result.failure : null, + artifacts: result ? result.artifacts : [], + resultId: runtimeTask.resultId, + }; +} + +function checkpointReceipt(entry) { + return { + checkpointId: entry.value.checkpointId, + taskId: entry.value.taskId, + attempt: entry.value.attempt, + status: entry.value.status, + startedAt: entry.value.startedAt, + completedAt: entry.value.completedAt, + failureClassification: entry.value.failureClassification, + interventionState: entry.value.interventionState, + reviewer: entry.value.reviewer, + result: entry.value.result, + }; +} + +function budgetReceipt(budget) { + const allocationChecks = Object.keys(budget.allocations).sort().map((name) => ({ + name, + approved: budget.allocations[name], + consumed: budget.consumed.allocations[name], + respected: budget.consumed.allocations[name] <= budget.allocations[name], + protected: budget.protectedAllocations.includes(name), + })); + const ceilings = { + modelOperations: budget.consumed.modelOperations <= budget.modelOperations, + targetedVerificationRuns: budget.consumed.targetedVerificationRuns <= budget.maxTargetedVerificationRuns, + fullVerificationRuns: budget.consumed.fullVerificationRuns <= budget.maxFullVerificationRuns, + capturedOutputBytes: budget.consumed.capturedOutputBytes <= budget.maxCapturedOutputBytes, + }; + const respected = Object.values(ceilings).every(Boolean) && allocationChecks.every((entry) => entry.respected); + return { + ...budget, + compliance: { + status: respected ? "passed" : "failed", + absoluteCeilingRespected: ceilings.modelOperations, + protectedAllocationsRespected: allocationChecks.filter((entry) => entry.protected).every((entry) => entry.respected), + ceilings, + allocations: allocationChecks, + }, + }; +} + +function buildEvidenceReceipt(found, options = {}) { + const generatedAt = options.generatedAt || new Date().toISOString(); + const warnings = []; + const results = readJsonDirectory(found, path.join(found.runRoot, "results"), "workflow result", warnings); + const reviews = readJsonDirectory(found, path.join(found.runRoot, "reviews"), "workflow review", warnings); + const checkpoints = readJsonDirectory(found, path.join(found.runRoot, "checkpoints"), "workflow checkpoint", warnings); + const events = readEvents(found, warnings); + const resultsByTask = new Map(results.map((entry) => [entry.value.taskId, entry.value])); + const reviewValues = reviews.map((entry) => entry.value); + const usageValues = [...results.map((entry) => entry.value), ...reviewValues].map((entry) => entry.usage); + let complete = found.run.status === "finalized" + && found.run.tasks.every((task) => task.status === "completed" && task.verification && task.verification.status === "passed") + && (!found.run.reviewerPolicy.requiredForFinalize || found.run.reviewer.status === "passed"); + if (!complete) warnings.push({ code: "run-not-finalized", message: `Run status ${found.run.status} produces a partial receipt.` }); + const integrityFiles = integrityInventory(found, results, reviews, warnings); + if (warnings.some((warning) => ["missing-referenced-evidence", "malformed-evidence-file", "malformed-event", "missing-events"].includes(warning.code))) { + complete = false; + } + const headCommit = getGitHeadCommit(found.repoRoot); + const baseCommit = found.run.git && found.run.git.baseCommit; + + return { + schemaVersion: EVIDENCE_RECEIPT_SCHEMA_VERSION, + generatedAt, + runId: found.run.runId, + goal: found.run.goal, + sourcePlan: found.run.approval.source, + workflow: found.run.workflow, + planRevisions: found.run.revisionHistory || [], + operatingModes: found.run.execution.allowedModes, + execution: { + owner: found.run.execution.owner, + backend: found.run.execution.backend, + }, + assurance: found.run.assurance, + completeness: { status: complete ? "complete" : "partial", runStatus: found.run.status }, + tasks: found.run.tasks.map((task) => taskReceipt(found, task, resultsByTask)), + checkpoints: checkpoints.map(checkpointReceipt), + interventions: found.run.interventions || [], + events, + providers: [{ + provider: found.run.execution.backend === "codex-exec" + ? { status: "known", value: "codex" } + : { status: "unknown", value: null, reason: "selected execution boundary does not identify a provider" }, + effectiveModel: { status: "unknown", value: null, reason: "workflow result does not expose an effective model" }, + }], + commands: found.definition.tasks.map((task) => ({ taskId: task.id, verification: task.verification })), + reviewer: found.run.reviewer, + reviews: reviewValues, + reviewHistory: found.run.reviewHistory || [], + budget: budgetReceipt(found.run.budget), + usage: { + managedOperations: truthAggregate(usageValues.map((usage) => usage && usage.managedOperations), "no uniformly observed managed operation evidence"), + capturedOutputBytes: truthAggregate(usageValues.map((usage) => usage && usage.capturedOutputBytes), "no uniformly observed captured output evidence"), + managedTokens: truthAggregate(usageValues.map((usage) => usage && usage.managedTokens), "managed token totals are unavailable"), + hostInternal: truthAggregate(usageValues.map((usage) => usage && usage.hostInternal), "host-internal usage is unavailable"), + }, + estimate: { schemaVersion: "usage-estimate/v1", label: "unknown", range: null, confidence: "unavailable", sampleBasis: null, driftState: "unknown" }, + cost: { apiEquivalent: { label: "unknown", value: null, currency: null, pricingDate: null, model: null, reason: "no valid dated API pricing mapping" } }, + warningSurface: { status: "unknown", deliveries: [] }, + git: { + baseCommit: baseCommit + ? { status: "known", value: baseCommit } + : { status: "unknown", value: null, reason: "historical run predates source git identity" }, + headCommit: { status: "known", value: headCommit }, + }, + policy: { + approval: found.run.approval, + assurance: found.run.assurance, + controls: loadIntegrationControlReceipt(found), + }, + integrity: { + algorithm: "sha256", + claim: "tamper-evident-local-metadata", + tamperProof: false, + files: integrityFiles, + sourceIdentities: { + workflowDigest: found.run.workflow.digest, + sourcePlanSha256: found.run.approval.source.sha256, + gitBaseCommit: baseCommit || null, + gitHeadCommit: headCommit, + }, + }, + timestamps: { + createdAt: found.run.createdAt, + updatedAt: found.run.updatedAt, + finalizedAt: found.run.finalization ? found.run.finalization.finalizedAt : null, + generatedAt, + }, + warnings, + }; +} + +function renderEvidenceReceiptMarkdown(receipt) { + const taskLines = receipt.tasks.map((task) => ( + `- ${task.id}: ${task.status}; attempts ${task.attempts}; scope ${task.scopeVerdict.status}; changed ${task.changedFiles.join(", ") || "none"}` + )).join("\n"); + const warningLines = receipt.warnings.length > 0 + ? receipt.warnings.map((warning) => `- ${warning.code}: ${warning.message}`).join("\n") + : "- none"; + return `# CEWP Evidence Receipt + +- Run: ${receipt.runId} +- Goal: ${receipt.goal} +- Status: ${receipt.completeness.runStatus} (${receipt.completeness.status}) +- Execution: ${receipt.execution.owner} / ${receipt.execution.backend || "none"} +- Reviewer: ${receipt.reviewer.decision || "none"} +- Source: ${receipt.sourcePlan.kind}; ${receipt.sourcePlan.path || "direct goal"} +- Git base: ${receipt.git.baseCommit.status === "known" ? receipt.git.baseCommit.value : "unknown"} +- Git head: ${receipt.git.headCommit.value} +- Managed operations: ${receipt.usage.managedOperations.label}${receipt.usage.managedOperations.value === null ? "" : ` (${receipt.usage.managedOperations.value})`} +- Host-internal usage: ${receipt.usage.hostInternal.label} +- API-equivalent cost: ${receipt.cost.apiEquivalent.label} +- Integrity: ${receipt.integrity.claim}; ${receipt.integrity.files.length} hashed files; not tamper-proof + +## Tasks + +${taskLines || "- none"} + +## Warnings + +${warningLines} + +This receipt excludes raw prompts and raw log contents by default. +`; +} + +function writeTextAtomic(filePath, content) { + const temporaryPath = `${filePath}.${process.pid}.tmp`; + fs.writeFileSync(temporaryPath, content, { flag: "wx" }); + try { + fs.renameSync(temporaryPath, filePath); + } catch (error) { + fs.rmSync(temporaryPath, { force: true }); + throw error; + } +} + +function writeEvidenceReceipt(found, options = {}) { + const receipt = buildEvidenceReceipt(found, options); + const jsonPath = path.join(found.runRoot, "evidence-receipt.json"); + const markdownPath = path.join(found.runRoot, "evidence-receipt.md"); + writeJsonAtomic(jsonPath, receipt); + writeTextAtomic(markdownPath, renderEvidenceReceiptMarkdown(receipt)); + return { receipt, paths: { json: jsonPath, markdown: markdownPath } }; +} + +module.exports = { + EVIDENCE_RECEIPT_SCHEMA_VERSION, + buildEvidenceReceipt, + renderEvidenceReceiptMarkdown, + writeEvidenceReceipt, +}; diff --git a/src/workflow/cli.js b/src/workflow/cli.js index 7abe7ba..583ac0a 100644 --- a/src/workflow/cli.js +++ b/src/workflow/cli.js @@ -30,6 +30,7 @@ const { } = require("./state"); const { deriveSchedule } = require("./scheduler"); const { listWorkflowTemplates, loadWorkflowTemplate } = require("./templates"); +const { writeEvidenceReceipt } = require("../evidence/receipt"); function outputJson(command, data) { console.log(JSON.stringify({ @@ -61,6 +62,20 @@ function resolveProposalSource(options) { } function runWorkflow(options = {}) { + if (options.subcommand === "receipt") { + if (!options.workflowRunId) throw new Error("workflow receipt requires a run id."); + const found = loadWorkflowRun(process.cwd(), options.workflowRunId); + const result = writeEvidenceReceipt(found); + if (options.json) outputJson("workflow.receipt", result); + else { + console.log("CEWP evidence receipt written"); + console.log(`Run ID: ${result.receipt.runId}`); + console.log(`Completeness: ${result.receipt.completeness.status}`); + console.log(`JSON: ${result.paths.json}`); + console.log(`Markdown: ${result.paths.markdown}`); + } + return; + } if (options.subcommand === "compile") { const request = createWorkflowCompilerRequest({ repoRoot: process.cwd(), diff --git a/src/workflow/state.js b/src/workflow/state.js index 3128d16..ae4db09 100644 --- a/src/workflow/state.js +++ b/src/workflow/state.js @@ -3,6 +3,7 @@ const fs = require("node:fs"); const path = require("node:path"); const { ensureDir } = require("../lib/fs"); +const { getGitHeadCommit } = require("../lib/git"); const { normalizeSlashPath } = require("../lib/paths"); const { digestWorkflowDefinition, validateWorkflowDefinition } = require("./definition"); const { readRepoJson } = require("./source"); @@ -1519,6 +1520,9 @@ function createApprovedRun(options) { status: "approved", createdAt: timestamp, updatedAt: timestamp, + git: { + baseCommit: getGitHeadCommit(repoRoot), + }, goal: options.definition.goal, execution: options.definition.execution, assurance: options.definition.assurance, diff --git a/tests/contracts/evidence-receipt.js b/tests/contracts/evidence-receipt.js new file mode 100644 index 0000000..f68239b --- /dev/null +++ b/tests/contracts/evidence-receipt.js @@ -0,0 +1,148 @@ +"use strict"; + +const fs = require("node:fs"); +const path = require("node:path"); +const { assert } = require("../harness/lib/assertions"); +const { cleanupRepo, makeTempRepo, runNode, writeFile } = require("../harness/lib/temp-repo"); +const { validDefinition } = require("./workflow-definition"); +const { successfulResult } = require("./workflow-result"); +const { approveWorkflow } = require("./workflow-scheduler"); +const { + finalizeWorkflowRun, + loadWorkflowRun, + recordWorkflowResult, + recordWorkflowReview, + startWorkflowTask, +} = require("../../src/workflow/state"); +const { + buildEvidenceReceipt, + renderEvidenceReceiptMarkdown, + writeEvidenceReceipt, +} = require("../../src/evidence/receipt"); + +const cewpCli = path.join(__dirname, "..", "..", "bin", "cewp.js"); + +function singleTaskDefinition() { + const definition = validDefinition(); + definition.workflowId = "evidence-receipt"; + definition.tasks = [definition.tasks[0]]; + definition.budget.maxTargetedVerificationRuns = 6; + definition.budget.maxFullVerificationRuns = 1; + return definition; +} + +function reviewerCandidate(run) { + return { + schemaVersion: "review-result/v1", + reviewId: `${run.runId}-final-review`, + runId: run.runId, + workflowDigest: run.workflow.digest, + scope: { kind: "workflow", taskId: null, checkpointId: null }, + completedAt: "2026-07-18T12:05:00.000Z", + independent: true, + decision: "PASS", + summary: "Independent evidence receipt review passed.", + findings: [], + evidence: [{ kind: "review-report", path: "evidence/review.md" }], + usage: { + managedOperations: { label: "observed", value: 1, source: "codex-exec-jsonl" }, + capturedOutputBytes: { label: "observed", value: 128, source: "cewp-bounded-output" }, + managedTokens: { label: "unknown", value: null, reason: "fixture omits token totals" }, + hostInternal: { label: "unknown", value: null, reason: "host usage is unavailable" }, + }, + }; +} + +function completeRun(repoRoot) { + const approved = approveWorkflow(repoRoot, singleTaskDefinition()); + let found = loadWorkflowRun(repoRoot, approved.runId); + const started = startWorkflowTask(found, "implement-example", { + now: new Date("2026-07-18T12:00:00.000Z"), + }); + writeFile(path.join(repoRoot, "evidence", "baseline.json"), "{\"status\":\"passed\"}\n"); + writeFile(path.join(repoRoot, "evidence", "targeted.json"), "{\"status\":\"passed\"}\n"); + writeFile(path.join(repoRoot, "evidence", "change.diff"), "diff --git a/src/example.js b/src/example.js\n"); + writeFile(path.join(repoRoot, "evidence", "review.md"), "# Independent review\n\nPASS\n"); + const result = successfulResult(approved, started.checkpoint, [{ + command: "node --test tests/example.test.js", + status: "passed", + evidencePath: "evidence/targeted.json", + }]); + found = loadWorkflowRun(repoRoot, approved.runId); + recordWorkflowResult(found, "implement-example", result); + found = loadWorkflowRun(repoRoot, approved.runId); + recordWorkflowReview(found, reviewerCandidate(approved)); + found = loadWorkflowRun(repoRoot, approved.runId); + finalizeWorkflowRun(found, { now: new Date("2026-07-18T12:06:00.000Z") }); + return loadWorkflowRun(repoRoot, approved.runId); +} + +function runContract() { + const repoRoot = makeTempRepo("cewp-evidence-receipt-"); + try { + const found = completeRun(repoRoot); + writeFile(path.join(found.runRoot, "adapter-output", "prompt.md"), "TOP_SECRET_PROMPT\n"); + const options = { generatedAt: "2026-07-18T12:07:00.000Z" }; + const first = buildEvidenceReceipt(found, options); + const second = buildEvidenceReceipt(found, options); + assert(first.schemaVersion === "evidence-receipt/v1", "evidence receipt is versioned"); + assert(JSON.stringify(first) === JSON.stringify(second), "fixed-timestamp receipt generation is deterministic"); + assert(first.completeness.status === "complete", "finalized reviewed run produces a complete receipt"); + assert(first.goal === found.run.goal, "receipt retains the approved goal"); + assert(first.sourcePlan.kind === "direct-goal", "receipt retains source-plan identity"); + assert(first.execution.owner === "managed" && first.execution.backend === "codex-exec", "receipt retains owner and backend"); + assert(first.tasks.length === 1 && first.checkpoints.length === 1, "receipt explains task and checkpoint structure"); + assert(first.tasks[0].changedFiles.includes("src/example.js"), "receipt records changed files"); + assert(first.tasks[0].verification.targeted[0].status === "passed", "receipt records verification evidence"); + assert(first.reviewer.decision === "PASS", "receipt retains independent reviewer PASS"); + assert(first.usage.managedOperations.label === "observed", "receipt aggregates observed managed operations"); + assert(first.usage.hostInternal.label === "unknown", "unavailable host usage remains unknown"); + assert(first.budget.compliance.status === "passed", "receipt proves the approved budget ceilings were respected"); + assert(first.budget.compliance.protectedAllocationsRespected === true, "receipt proves protected allocations were not overspent"); + assert(first.cost.apiEquivalent.label === "unknown", "currency cost is not invented"); + assert(first.integrity.claim === "tamper-evident-local-metadata", "integrity claim is explicitly local and tamper-evident"); + assert(first.integrity.files.length > 3, "receipt hashes canonical evidence files"); + assert(first.integrity.files.every((entry) => /^sha256:[a-f0-9]{64}$/.test(entry.sha256)), "evidence file hashes are stable sha256 values"); + assert(first.integrity.files.map((entry) => entry.path).join("\n") === first.integrity.files.map((entry) => entry.path).sort().join("\n"), "integrity inventory is sorted"); + assert(first.git.baseCommit.status === "known" && first.git.headCommit.status === "known", "receipt records source git identities"); + + const markdown = renderEvidenceReceiptMarkdown(first); + assert(markdown.includes("# CEWP Evidence Receipt"), "Markdown receipt is human-readable"); + assert(markdown.includes("Host-internal usage: unknown"), "Markdown keeps explicit unknowns"); + assert(!markdown.includes("TOP_SECRET_PROMPT"), "Markdown does not include prompt or raw log content"); + const written = writeEvidenceReceipt(found, options); + assert(fs.existsSync(written.paths.json) && fs.existsSync(written.paths.markdown), "JSON and Markdown receipts are written locally"); + assert(JSON.stringify(written.receipt) === JSON.stringify(first), "written receipt uses the same normalized model"); + const cli = runNode(cewpCli, ["workflow", "receipt", found.run.runId, "--json"], repoRoot); + assert(cli.status === 0, `workflow receipt CLI succeeds: ${cli.stderr}`); + const cliOutput = JSON.parse(cli.stdout); + assert(cliOutput.command === "workflow.receipt", "receipt output uses operator JSON"); + assert(cliOutput.data.receipt.schemaVersion === "evidence-receipt/v1", "CLI writes the normalized receipt contract"); + + fs.rmSync(path.join(repoRoot, "evidence", "change.diff")); + const missingEvidence = buildEvidenceReceipt(found, options); + assert(missingEvidence.completeness.status === "partial", "missing referenced evidence closes receipt completeness"); + assert(missingEvidence.warnings.some((warning) => warning.code === "missing-referenced-evidence"), "missing evidence has an actionable warning"); + + const partialRepo = makeTempRepo("cewp-evidence-receipt-partial-"); + try { + const partialRun = approveWorkflow(partialRepo, singleTaskDefinition()); + const partial = buildEvidenceReceipt(loadWorkflowRun(partialRepo, partialRun.runId), options); + assert(partial.completeness.status === "partial", "incomplete historical run produces a partial receipt"); + assert(partial.warnings.some((warning) => warning.code === "run-not-finalized"), "partial receipt explains why it is incomplete"); + } finally { + cleanupRepo(partialRepo); + } + } finally { + cleanupRepo(repoRoot); + } +} + +try { + runContract(); + console.log("[PASS] normalized evidence receipt is deterministic and tamper-evident"); +} catch (error) { + console.error("[FAIL] evidence receipt contract"); + console.error(error && error.stack ? error.stack : error); + process.exitCode = 1; +} diff --git a/tests/contracts/workflow-release.js b/tests/contracts/workflow-release.js index 839b8d1..5405ce3 100644 --- a/tests/contracts/workflow-release.js +++ b/tests/contracts/workflow-release.js @@ -22,7 +22,7 @@ function runWorkflowReleaseContract() { assert(unreleasedIndex >= 0 && releaseIndex > unreleasedIndex, "fresh Unreleased precedes the Phase 11 release"); assert(previousIndex > releaseIndex, "Phase 11 release precedes earlier release history"); const unreleased = releaseNotes.slice(unreleasedIndex, releaseIndex); - assert(unreleased.includes("No changes yet."), "fresh Unreleased is explicitly empty"); + assert(unreleased.includes("Phase 12 development"), "post-Phase 11 work remains in Unreleased"); for (const claim of [ "native and managed ownership", "no automatic model routing", From 71df53e6db5e5ce05c3bf398f9c93456dca96f27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tuncay=20=C3=96lmez?= Date: Thu, 23 Jul 2026 12:30:00 +0300 Subject: [PATCH 18/40] Add versioned event health verification --- bin/cewp.js | 6 + docs/evidence-receipts.md | 12 ++ docs/release-notes.md | 2 + package.json | 5 +- src/cli/parse.js | 2 +- src/cli/usage.js | 1 + src/evidence/events.js | 133 +++++++++++++++++++ src/evidence/receipt.js | 14 +- src/evidence/verify.js | 191 ++++++++++++++++++++++++++++ src/integration/hook-evidence.js | 6 +- src/run/verify.js | 21 +++ src/workflow/migration.js | 6 +- src/workflow/state.js | 8 +- tests/contracts/event-run-verify.js | 110 ++++++++++++++++ 14 files changed, 496 insertions(+), 21 deletions(-) create mode 100644 src/evidence/events.js create mode 100644 src/evidence/verify.js create mode 100644 src/run/verify.js create mode 100644 tests/contracts/event-run-verify.js diff --git a/bin/cewp.js b/bin/cewp.js index c5c2af2..63325e2 100644 --- a/bin/cewp.js +++ b/bin/cewp.js @@ -19,6 +19,7 @@ const { } = require("../src/run/basic"); const { runFinalize } = require("../src/run/finalize"); const { runCollect } = require("../src/run/collect"); +const { runVerify } = require("../src/run/verify"); const { runDispatchPlan } = require("../src/run/dispatch/plan"); const { runDispatchCheck } = require("../src/run/dispatch/check"); const { runDispatchPrompts } = require("../src/run/dispatch/prompts"); @@ -77,6 +78,11 @@ async function runCommand(options) { return; } + if (options.subcommand === "verify") { + runVerify(options); + return; + } + if (options.subcommand === "prompts") { runPrompts(options); return; diff --git a/docs/evidence-receipts.md b/docs/evidence-receipts.md index 32addbd..c30d1a6 100644 --- a/docs/evidence-receipts.md +++ b/docs/evidence-receipts.md @@ -37,3 +37,15 @@ Raw prompts, adapter output, transcripts, and raw log contents are excluded by d contain repository-relative paths, changed filenames, approved commands, bounded failure/review summaries, and artifact names. Inspect these fields before sharing. Phase 12 privacy work will add export redaction; until then, keep the receipt local if those metadata fields are sensitive. + +## Event Ledger And Run Health + +New workflow lifecycle records use `event/v1`, with a closed type-to-category vocabulary covering run, +revision, task, checkpoint, dispatch, intervention, verification, usage, estimates, budgets, warnings, +pauses, scope, review, cancellation, and finalization. Existing `workflow-event/v1` records are accepted as +read-only legacy input and normalized in receipts; CEWP does not rewrite historical ledgers implicitly. + +`cewp run verify ` checks canonical state/definition consistency, event syntax and schemas, +required result/checkpoint artifacts, bound-worktree liveness, and every receipt integrity hash. It executes +no agent and no approved verification command. A failed check returns a nonzero exit code while `--json` +still emits the complete `run-verification/v1` diagnostic model. diff --git a/docs/release-notes.md b/docs/release-notes.md index 3507a7b..7e07aab 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -6,6 +6,8 @@ - Added deterministic `evidence-receipt/v1` JSON/Markdown generation for workflow runs with complete-versus-partial truth, task/checkpoint/review evidence, usage unknowns, budget compliance, git identities, and local SHA-256 integrity metadata. - Added `cewp workflow receipt ` without executing agents or verification commands. Raw prompts, transcripts, adapter output, and raw log contents remain excluded by default. +- Added the closed `event/v1` lifecycle vocabulary with read-only normalization of historical `workflow-event/v1` records. +- Added read-only `cewp run verify ` checks for state, schemas, events, required artifacts, worktree liveness, and receipt integrity. ## 0.11.0-beta.0 diff --git a/package.json b/package.json index b6c0055..54239bc 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,7 @@ }, "scripts": { "test": "npm run test:contracts && npm run test:phase12 && npm run smoke", - "test:phase12": "npm run test:evidence-receipt", + "test:phase12": "npm run test:evidence-receipt && npm run test:event-run-verify", "test:contracts": "npm run test:init-install && npm run test:adapter-profile && npm run test:doctor-json && npm run test:operator-json && npm run test:hook-output && npm run test:skill-format && npm run test:ownership-gates && npm run test:fixtures && npm run test:plugin-package && npm run test:supervised-intake && npm run test:supervised-proposal && npm run test:supervised-controls && npm run test:supervised-execution && npm run test:supervised-review && npm run test:supervised-failure && npm run test:supervised-linear-resume && npm run test:supervised-demo && npm run test:workflow-definition && npm run test:workflow-compiler && npm run test:workflow-proposal && npm run test:workflow-scheduler && npm run test:workflow-worker-matrix && npm run test:workflow-result && npm run test:workflow-failure-result && npm run test:workflow-state-machine && npm run test:workflow-interventions && npm run test:workflow-failure-matrix && npm run test:workflow-budget && npm run test:workflow-progress && npm run test:workflow-review && npm run test:workflow-checkpoint-review && npm run test:workflow-lifecycle && npm run test:workflow-revision && npm run test:workflow-migration && npm run test:workflow-templates && npm run test:workflow-docs && npm run test:workflow-release && npm run test:integration-capabilities && npm run test:integration-binding && npm run test:integration-effort-policy && npm run test:integration-hook-evidence && npm run test:integration-mcp && npm run test:integration-observation && npm run test:native-goal-events", "test:init-install": "node tests/contracts/init-install.js", "test:adapter-profile": "node tests/contracts/adapter-profile.js", @@ -98,6 +98,7 @@ "test:integration-hook-evidence": "node tests/contracts/integration-hook-evidence.js", "test:integration-mcp": "node tests/contracts/integration-mcp.js", "test:evidence-receipt": "node tests/contracts/evidence-receipt.js", + "test:event-run-verify": "node tests/contracts/event-run-verify.js", "test:integration-observation": "node tests/contracts/integration-observation.js", "test:native-goal-events": "node tests/contracts/native-goal-events.js", "demo:supervised": "node src/demo/supervised.js", @@ -109,7 +110,7 @@ "check:syntax": "node --check ./bin/cewp.js && node --check ./src/demo/fake-codex.js && node --check ./src/demo/supervised.js && node --check ./src/integration/capabilities.js && node --check ./src/integration/binding.js && node --check ./src/integration/cli.js && node --check ./src/integration/effort-policy.js && node --check ./src/integration/hook-evidence.js && node --check ./src/integration/observation.js && node --check ./src/integration/native-goal.js && node --check ./src/skills/install.js && node --check ./src/skills/format.js && node --check ./src/skills/doctor-report.js && node --check ./src/run/ownership.js && node --check ./src/run/control-gates.js && node --check ./src/supervise/budget.js && node --check ./src/supervise/cli.js && node --check ./src/supervise/commands.js && node --check ./src/supervise/controls.js && node --check ./src/supervise/execution.js && node --check ./src/supervise/profiles.js && node --check ./src/supervise/receipt.js && node --check ./src/supervise/review.js && node --check ./src/supervise/state.js && node --check ./src/supervise/test-authoring.js && node --check ./src/supervise/verification.js && node --check ./src/workflow/cli.js && node --check ./src/workflow/definition.js && node --check ./src/workflow/graph.js && node --check ./src/workflow/result.js && node --check ./src/workflow/scheduler.js && node --check ./src/workflow/source.js && node --check ./src/workflow/state.js && node --check ./src/workflow/transitions.js && node --check ./plugins/cewp/hooks/capture-subagent.js && node --check ./tests/contracts/init-install.js && node --check ./tests/contracts/adapter-profile.js && node --check ./tests/contracts/doctor-json.js && node --check ./tests/contracts/operator-json.js && node --check ./tests/contracts/hook-output.js && node --check ./tests/contracts/skill-format.js && node --check ./tests/contracts/ownership-gates.js && node --check ./tests/contracts/deterministic-fixtures.js && node --check ./tests/contracts/plugin-package.js && node --check ./tests/contracts/supervised-intake.js && node --check ./tests/contracts/supervised-proposal.js && node --check ./tests/contracts/supervised-controls.js && node --check ./tests/contracts/supervised-execution.js && node --check ./tests/contracts/supervised-review.js && node --check ./tests/contracts/supervised-failure.js && node --check ./tests/contracts/supervised-linear-resume.js && node --check ./tests/contracts/supervised-demo.js && node --check ./tests/contracts/integration-capabilities.js && node --check ./tests/contracts/integration-binding.js && node --check ./tests/contracts/integration-effort-policy.js && node --check ./tests/contracts/integration-hook-evidence.js && node --check ./tests/contracts/integration-observation.js && node --check ./tests/contracts/native-goal-events.js && node --check ./tests/contracts/workflow-definition.js && node --check ./tests/contracts/workflow-proposal.js && node --check ./tests/contracts/workflow-scheduler.js && node --check ./tests/contracts/workflow-result.js && node --check ./tests/contracts/workflow-state-machine.js && node --check ./tests/contracts/workflow-interventions.js && node --check ./tests/capabilities/clean-install.js && node --check ./tests/capabilities/plugin-lifecycle.js && node --check ./tests/capabilities/codex-app-server.js && node --check ./tests/capabilities/fixtures/deny-shell-hook.js", "check:workflow-syntax": "node --check ./src/workflow/budget.js && node --check ./src/workflow/compiler.js && node --check ./src/workflow/migration.js && node --check ./src/workflow/progress.js && node --check ./src/workflow/proposal.js && node --check ./src/workflow/review.js && node --check ./src/workflow/revision.js && node --check ./src/workflow/templates.js && node --check ./tests/contracts/workflow-budget.js && node --check ./tests/contracts/workflow-checkpoint-review.js && node --check ./tests/contracts/workflow-compiler.js && node --check ./tests/contracts/workflow-docs.js && node --check ./tests/contracts/workflow-failure-matrix.js && node --check ./tests/contracts/workflow-failure-result.js && node --check ./tests/contracts/workflow-migration.js && node --check ./tests/contracts/workflow-progress.js && node --check ./tests/contracts/workflow-release.js && node --check ./tests/contracts/workflow-review.js && node --check ./tests/contracts/workflow-lifecycle.js && node --check ./tests/contracts/workflow-revision.js && node --check ./tests/contracts/workflow-templates.js && node --check ./tests/contracts/workflow-worker-matrix.js", "check:mcp": "node --check ./bin/cewp-mcp.js && node --check ./src/mcp/server.js && node --check ./src/mcp/tools.js && node --check ./tests/contracts/integration-mcp.js", - "check:evidence": "node --check ./src/evidence/receipt.js && node --check ./tests/contracts/evidence-receipt.js", + "check:evidence": "node --check ./src/evidence/receipt.js && node --check ./src/evidence/events.js && node --check ./src/evidence/verify.js && node --check ./src/run/verify.js && node --check ./tests/contracts/evidence-receipt.js && node --check ./tests/contracts/event-run-verify.js", "check": "npm run check:syntax && npm run check:workflow-syntax && npm run check:mcp && npm run check:evidence && npm test", "pack:dry-run": "npm pack --dry-run" }, diff --git a/src/cli/parse.js b/src/cli/parse.js index b76bf5f..872ae2c 100644 --- a/src/cli/parse.js +++ b/src/cli/parse.js @@ -107,7 +107,7 @@ function parseArgs(argv) { continue; } - if (args.command === "run" && ["status", "next", "resume"].includes(args.subcommand) && index === 2 && !arg.startsWith("--")) { + if (args.command === "run" && ["status", "next", "resume", "verify"].includes(args.subcommand) && index === 2 && !arg.startsWith("--")) { args.runId = arg; continue; } diff --git a/src/cli/usage.js b/src/cli/usage.js index 2283f04..7881e0e 100644 --- a/src/cli/usage.js +++ b/src/cli/usage.js @@ -57,6 +57,7 @@ Usage: cewp run status [run-id] [--json] cewp run next [run-id] [--json] cewp run resume [run-id] [--json] + cewp run verify [--json] cewp run prompts cewp run prompt cewp run worktrees plan diff --git a/src/evidence/events.js b/src/evidence/events.js new file mode 100644 index 0000000..cdc5307 --- /dev/null +++ b/src/evidence/events.js @@ -0,0 +1,133 @@ +"use strict"; + +const fs = require("node:fs"); + +const EVENT_SCHEMA_VERSION = "event/v1"; +const LEGACY_EVENT_SCHEMA_VERSIONS = new Set(["workflow-event/v1"]); +const EVENT_CATEGORIES = Object.freeze({ + "workflow-approved": "run", + "workflow-migrated": "plan-revision", + "workflow-revised": "plan-revision", + "task-started": "task", + "task-failed": "task", + "task-completed": "task", + "checkpoint-started": "checkpoint", + "checkpoint-completed": "checkpoint", + "dispatch-started": "dispatch", + "dispatch-completed": "dispatch", + "workflow-intervention": "operator-intervention", + "verification-completed": "verification", + "usage-observed": "usage-observation", + "estimate-revised": "estimate-revision", + "budget-approved": "budget-approval", + "allocation-consumed": "allocation-consumption", + "budget-threshold": "threshold", + "warning-presented": "warning-presentation", + "pause-budget-safe": "safe-pause", + "paused-budget-safe": "safe-pause", + "pause-budget-unverified": "unverified-pause", + "paused-budget-unverified": "unverified-pause", + "pause-host-limit": "host-limit", + "paused-host-limit": "host-limit", + "scope-evaluated": "scope", + "review-passed": "review", + "review-blocked": "review", + "checkpoint-review-passed": "review", + "checkpoint-review-blocked": "review", + "workflow-cancelled": "cancellation", + "workflow-finalized": "finalize", + "workflow-lifecycle": "run", + "hook-evidence-approved": "scope", +}); + +function isObject(value) { + return Boolean(value) && typeof value === "object" && !Array.isArray(value); +} + +function normalizeLifecycleEvent(candidate, options = {}) { + if (!isObject(candidate)) throw new Error("Lifecycle event must be an object."); + const legacy = LEGACY_EVENT_SCHEMA_VERSIONS.has(candidate.schemaVersion); + if (candidate.schemaVersion !== EVENT_SCHEMA_VERSION && !(options.allowLegacy && legacy)) { + throw new Error(`Incompatible lifecycle event schema: ${candidate.schemaVersion || "missing"}.`); + } + if (typeof candidate.type !== "string" || !EVENT_CATEGORIES[candidate.type]) { + throw new Error(`Unknown lifecycle event type: ${candidate.type || "missing"}.`); + } + if (typeof candidate.runId !== "string" || candidate.runId.length === 0) { + throw new Error("Lifecycle event runId is required."); + } + if (options.runId && candidate.runId !== options.runId) { + throw new Error(`Lifecycle event runId mismatch: ${candidate.runId}.`); + } + if (typeof candidate.timestamp !== "string" || !Number.isFinite(Date.parse(candidate.timestamp))) { + throw new Error("Lifecycle event timestamp must be an ISO timestamp."); + } + const category = EVENT_CATEGORIES[candidate.type]; + if (!legacy && candidate.category !== category) { + throw new Error(`Lifecycle event ${candidate.type} requires category ${category}.`); + } + return { + ...candidate, + schemaVersion: EVENT_SCHEMA_VERSION, + timestamp: new Date(candidate.timestamp).toISOString(), + category, + }; +} + +function createLifecycleEvent(candidate) { + return normalizeLifecycleEvent({ + ...candidate, + schemaVersion: EVENT_SCHEMA_VERSION, + category: EVENT_CATEGORIES[candidate.type], + }); +} + +function parseLifecycleEvents(content, options = {}) { + const events = []; + const issues = []; + String(content).split(/\r?\n/).forEach((line, index) => { + if (!line) return; + let parsed; + try { + parsed = JSON.parse(line); + } catch { + issues.push({ code: "malformed-event", line: index + 1, message: "Lifecycle event is not valid JSON." }); + return; + } + try { + events.push(normalizeLifecycleEvent(parsed, { ...options, allowLegacy: true })); + } catch (error) { + issues.push({ + code: /schema/.test(error.message) ? "incompatible-event-schema" : "invalid-event", + line: index + 1, + message: error.message, + }); + } + }); + return { events, issues }; +} + +function readLifecycleEvents(filePath, options = {}) { + const parsed = parseLifecycleEvents(fs.readFileSync(filePath, "utf8"), options); + if (parsed.issues.length > 0) { + const first = parsed.issues[0]; + throw new Error(`${first.code} at line ${first.line}: ${first.message}`); + } + return parsed.events; +} + +function appendLifecycleEvent(runRoot, candidate) { + const event = createLifecycleEvent(candidate); + fs.appendFileSync(require("node:path").join(runRoot, "events.jsonl"), `${JSON.stringify(event)}\n`); + return event; +} + +module.exports = { + EVENT_CATEGORIES, + EVENT_SCHEMA_VERSION, + appendLifecycleEvent, + createLifecycleEvent, + normalizeLifecycleEvent, + parseLifecycleEvents, + readLifecycleEvents, +}; diff --git a/src/evidence/receipt.js b/src/evidence/receipt.js index c33b872..d6d35f5 100644 --- a/src/evidence/receipt.js +++ b/src/evidence/receipt.js @@ -6,6 +6,7 @@ const path = require("node:path"); const { getGitHeadCommit } = require("../lib/git"); const { normalizeSlashPath } = require("../lib/paths"); const { loadIntegrationControlReceipt } = require("../integration/binding"); +const { parseLifecycleEvents } = require("./events"); const { writeJsonAtomic } = require("../workflow/state"); const EVIDENCE_RECEIPT_SCHEMA_VERSION = "evidence-receipt/v1"; @@ -66,14 +67,9 @@ function readEvents(found, warnings) { warnings.push({ code: "missing-events", message: "Workflow event ledger is missing." }); return []; } - return fs.readFileSync(eventPath, "utf8").split(/\r?\n/).filter(Boolean).map((line, index) => { - try { - return JSON.parse(line); - } catch { - warnings.push({ code: "malformed-event", line: index + 1, message: "Workflow event is not valid JSON." }); - return { schemaVersion: "event/v1", type: "malformed", line: index + 1 }; - } - }); + const parsed = parseLifecycleEvents(fs.readFileSync(eventPath, "utf8"), { runId: found.run.runId }); + warnings.push(...parsed.issues); + return parsed.events; } function truthAggregate(values, reason) { @@ -228,7 +224,7 @@ function buildEvidenceReceipt(found, options = {}) { && (!found.run.reviewerPolicy.requiredForFinalize || found.run.reviewer.status === "passed"); if (!complete) warnings.push({ code: "run-not-finalized", message: `Run status ${found.run.status} produces a partial receipt.` }); const integrityFiles = integrityInventory(found, results, reviews, warnings); - if (warnings.some((warning) => ["missing-referenced-evidence", "malformed-evidence-file", "malformed-event", "missing-events"].includes(warning.code))) { + if (warnings.some((warning) => ["missing-referenced-evidence", "malformed-evidence-file", "malformed-event", "incompatible-event-schema", "invalid-event", "missing-events"].includes(warning.code))) { complete = false; } const headCommit = getGitHeadCommit(found.repoRoot); diff --git a/src/evidence/verify.js b/src/evidence/verify.js new file mode 100644 index 0000000..5c0ff8c --- /dev/null +++ b/src/evidence/verify.js @@ -0,0 +1,191 @@ +"use strict"; + +const crypto = require("node:crypto"); +const fs = require("node:fs"); +const path = require("node:path"); +const { normalizeSlashPath } = require("../lib/paths"); +const { loadWorkflowRun } = require("../workflow/state"); +const { parseLifecycleEvents } = require("./events"); +const { buildEvidenceReceipt } = require("./receipt"); + +const RUN_VERIFICATION_SCHEMA_VERSION = "run-verification/v1"; + +function sha256(filePath) { + return `sha256:${crypto.createHash("sha256").update(fs.readFileSync(filePath)).digest("hex")}`; +} + +function isInside(parentPath, childPath) { + const relative = path.relative(path.resolve(parentPath), path.resolve(childPath)); + return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative)); +} + +function check(id, issues, startIndex, skipped = false) { + const ownIssues = issues.slice(startIndex); + return { + id, + status: skipped ? "not-applicable" : ownIssues.some((entry) => entry.severity === "error") ? "failed" : "passed", + issueCount: ownIssues.length, + }; +} + +function issue(issues, code, message, details = {}) { + issues.push({ severity: "error", code, message, ...details }); +} + +function readJsonRecords(rootPath, issues, code) { + if (!fs.existsSync(rootPath)) return []; + return fs.readdirSync(rootPath, { withFileTypes: true }) + .filter((entry) => entry.isFile() && entry.name.endsWith(".json")) + .map((entry) => { + const filePath = path.join(rootPath, entry.name); + try { + return JSON.parse(fs.readFileSync(filePath, "utf8")); + } catch (error) { + issue(issues, code, `Artifact is not valid JSON: ${normalizeSlashPath(path.relative(rootPath, filePath))}: ${error.message}`); + return null; + } + }) + .filter(Boolean); +} + +function verifyReceipt(found, issues) { + const receiptPath = path.join(found.runRoot, "evidence-receipt.json"); + if (!fs.existsSync(receiptPath)) { + if (found.run.status === "finalized") { + issue(issues, "missing-evidence-receipt", "Finalized workflow run has no evidence receipt."); + } + return false; + } + let receipt; + try { + receipt = JSON.parse(fs.readFileSync(receiptPath, "utf8")); + } catch (error) { + issue(issues, "malformed-evidence-receipt", `Evidence receipt is not valid JSON: ${error.message}`); + return true; + } + if (receipt.schemaVersion !== "evidence-receipt/v1" || receipt.runId !== found.run.runId) { + issue(issues, "incompatible-evidence-receipt", "Evidence receipt schema or run identity is incompatible."); + return true; + } + const files = receipt.integrity && receipt.integrity.files; + if (!Array.isArray(files)) { + issue(issues, "malformed-evidence-receipt", "Evidence receipt integrity inventory is missing."); + return true; + } + try { + const rebuilt = buildEvidenceReceipt(found, { generatedAt: receipt.generatedAt }); + if (JSON.stringify(rebuilt.integrity) !== JSON.stringify(receipt.integrity)) { + issue(issues, "receipt-integrity-inventory-mismatch", "Receipt integrity inventory does not match current canonical evidence."); + } + } catch (error) { + issue(issues, "receipt-integrity-rebuild-failed", `Could not rebuild canonical receipt integrity: ${error.message}`); + } + for (const entry of files) { + const filePath = typeof entry.path === "string" ? path.resolve(found.repoRoot, entry.path) : ""; + if (!filePath || !isInside(found.repoRoot, filePath) || !fs.existsSync(filePath) || !fs.statSync(filePath).isFile()) { + issue(issues, "receipt-integrity-missing", `Receipt evidence is unavailable: ${entry.path || "missing path"}.`, { path: entry.path || null }); + continue; + } + const actual = sha256(filePath); + const bytes = fs.statSync(filePath).size; + if (actual !== entry.sha256 || bytes !== entry.bytes) { + issue(issues, "receipt-integrity-mismatch", `Receipt evidence changed: ${entry.path}.`, { + path: entry.path, + expectedSha256: entry.sha256, + actualSha256: actual, + }); + } + } + return true; +} + +function verifyBoundWorktree(found, issues) { + const bindingPath = path.join(found.runRoot, "integration", "host-binding.json"); + if (!fs.existsSync(bindingPath)) return false; + let binding; + try { + binding = JSON.parse(fs.readFileSync(bindingPath, "utf8")); + } catch (error) { + issue(issues, "malformed-host-binding", `Host binding is not valid JSON: ${error.message}`); + return true; + } + const worktree = binding.references && binding.references.worktree; + if (worktree && typeof worktree.path === "string" && !fs.existsSync(path.resolve(found.repoRoot, worktree.path))) { + issue(issues, "stale-worktree", `Bound worktree is unavailable: ${normalizeSlashPath(worktree.path)}.`, { + worktreeId: worktree.id || null, + }); + } + return true; +} + +function verifyWorkflowRun(repoRoot, runId) { + const issues = []; + const checks = []; + let found; + let start = issues.length; + try { + found = loadWorkflowRun(repoRoot, runId); + } catch (error) { + issue(issues, "state-inconsistent", error.message); + } + checks.push(check("state-consistency", issues, start)); + if (!found) { + return { + schemaVersion: RUN_VERIFICATION_SCHEMA_VERSION, + runId, + status: "failed", + checks, + issues, + execution: { agentsExecuted: false, verificationCommandsExecuted: false }, + }; + } + + start = issues.length; + const eventsPath = path.join(found.runRoot, "events.jsonl"); + if (!fs.existsSync(eventsPath)) { + issue(issues, "missing-events", "Workflow event ledger is missing."); + } else { + const parsed = parseLifecycleEvents(fs.readFileSync(eventsPath, "utf8"), { runId }); + for (const entry of parsed.issues) issue(issues, entry.code, entry.message, { line: entry.line }); + } + checks.push(check("event-ledger", issues, start)); + + start = issues.length; + for (const task of found.run.tasks) { + if (task.resultId) { + const results = readJsonRecords(path.join(found.runRoot, "results", task.id), issues, "malformed-task-result"); + if (!results.some((entry) => entry.resultId === task.resultId)) { + issue(issues, "missing-task-result", `Task ${task.id} references result ${task.resultId}, but that result is missing.`, { taskId: task.id }); + } + } + if (task.activeCheckpointId) { + const checkpoints = readJsonRecords(path.join(found.runRoot, "checkpoints", task.id), issues, "malformed-checkpoint"); + if (!checkpoints.some((entry) => entry.checkpointId === task.activeCheckpointId)) { + issue(issues, "missing-checkpoint", `Task ${task.id} active checkpoint ${task.activeCheckpointId} is missing.`, { taskId: task.id }); + } + } + } + checks.push(check("required-artifacts", issues, start)); + + start = issues.length; + const hadBinding = verifyBoundWorktree(found, issues); + checks.push(check("worktree-liveness", issues, start, !hadBinding)); + + start = issues.length; + const hadReceipt = verifyReceipt(found, issues); + checks.push(check("receipt-integrity", issues, start, !hadReceipt)); + + return { + schemaVersion: RUN_VERIFICATION_SCHEMA_VERSION, + runId, + status: issues.some((entry) => entry.severity === "error") ? "failed" : "passed", + checks, + issues, + execution: { agentsExecuted: false, verificationCommandsExecuted: false }, + }; +} + +module.exports = { + RUN_VERIFICATION_SCHEMA_VERSION, + verifyWorkflowRun, +}; diff --git a/src/integration/hook-evidence.js b/src/integration/hook-evidence.js index c20dd76..0ca75c2 100644 --- a/src/integration/hook-evidence.js +++ b/src/integration/hook-evidence.js @@ -4,6 +4,7 @@ const childProcess = require("node:child_process"); const crypto = require("node:crypto"); const fs = require("node:fs"); const path = require("node:path"); +const { appendLifecycleEvent } = require("../evidence/events"); const { loadWorkflowRun } = require("../workflow/state"); const { writeJsonAtomic } = require("../workflow/state"); @@ -293,15 +294,14 @@ function approveCodexHookTrust(options = {}) { trustDigest: trust.approval.digest, activatedAt: approvedAt, }); - fs.appendFileSync(path.join(found.runRoot, "events.jsonl"), `${JSON.stringify({ - schemaVersion: "workflow-event/v1", + appendLifecycleEvent(found.runRoot, { timestamp: approvedAt, type: "hook-evidence-approved", runId: found.run.runId, revision: found.run.workflow.revision, actor: "operator", approvalDigest: trust.approval.digest, - })}\n`); + }); return { trust, nextAction: { diff --git a/src/run/verify.js b/src/run/verify.js new file mode 100644 index 0000000..4b4ec55 --- /dev/null +++ b/src/run/verify.js @@ -0,0 +1,21 @@ +"use strict"; + +const { verifyWorkflowRun } = require("../evidence/verify"); + +function runVerify(options = {}) { + if (!options.runId) throw new Error("run verify requires a workflow run id."); + const report = verifyWorkflowRun(process.cwd(), options.runId); + if (options.json) { + console.log(JSON.stringify({ schemaVersion: "operator-json/v1", command: "run.verify", data: report }, null, 2)); + } else { + console.log("CEWP run verification"); + console.log(`Run ID: ${report.runId}`); + console.log(`Status: ${report.status}`); + for (const entry of report.checks) console.log(`${entry.id}: ${entry.status}`); + for (const entry of report.issues) console.log(`[${entry.code}] ${entry.message}`); + } + if (report.status !== "passed") process.exitCode = 1; + return report; +} + +module.exports = { runVerify }; diff --git a/src/workflow/migration.js b/src/workflow/migration.js index a0bafd3..ca10852 100644 --- a/src/workflow/migration.js +++ b/src/workflow/migration.js @@ -5,6 +5,7 @@ const fs = require("node:fs"); const path = require("node:path"); const { ensureDir } = require("../lib/fs"); const { normalizeSlashPath } = require("../lib/paths"); +const { appendLifecycleEvent } = require("../evidence/events"); const { findSupervisedRun } = require("../supervise/state"); const { digestWorkflowDefinition, @@ -366,8 +367,7 @@ function applyLegacyMigration(repoRoot, runId, options = {}) { }; writeJsonAtomic(runPath, run); const progress = writeWorkflowProgress(runRoot, run, preview.definition, { now: new Date(timestamp) }); - fs.appendFileSync(path.join(runRoot, "events.jsonl"), `${JSON.stringify({ - schemaVersion: "workflow-event/v1", + appendLifecycleEvent(runRoot, { timestamp, type: "workflow-migrated", runId: run.runId, @@ -378,7 +378,7 @@ function applyLegacyMigration(repoRoot, runId, options = {}) { migrationDigest: preview.migrationDigest, backupPath: normalizeSlashPath(path.relative(preview.found.repoRoot, backupPath)), actor: "operator", - })}\n`); + }); const record = { schemaVersion: "workflow-migration/v1", projectionVersion: MIGRATION_PROJECTION_VERSION, diff --git a/src/workflow/state.js b/src/workflow/state.js index ae4db09..685c0a3 100644 --- a/src/workflow/state.js +++ b/src/workflow/state.js @@ -4,6 +4,7 @@ const fs = require("node:fs"); const path = require("node:path"); const { ensureDir } = require("../lib/fs"); const { getGitHeadCommit } = require("../lib/git"); +const { appendLifecycleEvent } = require("../evidence/events"); const { normalizeSlashPath } = require("../lib/paths"); const { digestWorkflowDefinition, validateWorkflowDefinition } = require("./definition"); const { readRepoJson } = require("./source"); @@ -320,7 +321,7 @@ function loadWorkflowRun(repoRoot, runId) { } function appendWorkflowEvent(runRoot, event) { - fs.appendFileSync(path.join(runRoot, "events.jsonl"), `${JSON.stringify(event)}\n`); + appendLifecycleEvent(runRoot, event); } function pauseForWorkflowBudget(found, allocation, decision, timestamp) { @@ -1560,7 +1561,8 @@ function createApprovedRun(options) { warnings: [], }; writeJsonAtomic(runPath, run); - fs.writeFileSync(path.join(runRoot, "events.jsonl"), `${JSON.stringify({ + fs.writeFileSync(path.join(runRoot, "events.jsonl"), "", { flag: "wx" }); + appendWorkflowEvent(runRoot, { schemaVersion: "workflow-event/v1", timestamp, type: "workflow-approved", @@ -1570,7 +1572,7 @@ function createApprovedRun(options) { digest, approvalDigest, actor: "operator", - })}\n`, { flag: "wx" }); + }); writeWorkflowProgress(runRoot, run, options.definition, { now }); return { run, diff --git a/tests/contracts/event-run-verify.js b/tests/contracts/event-run-verify.js new file mode 100644 index 0000000..4ad9456 --- /dev/null +++ b/tests/contracts/event-run-verify.js @@ -0,0 +1,110 @@ +"use strict"; + +const crypto = require("node:crypto"); +const fs = require("node:fs"); +const path = require("node:path"); +const { assert } = require("../harness/lib/assertions"); +const { cleanupRepo, makeTempRepo, runNode, writeFile } = require("../harness/lib/temp-repo"); +const { validDefinition } = require("./workflow-definition"); +const { approveWorkflow } = require("./workflow-scheduler"); +const { loadWorkflowRun, startWorkflowTask } = require("../../src/workflow/state"); +const { writeEvidenceReceipt } = require("../../src/evidence/receipt"); +const { + EVENT_SCHEMA_VERSION, + EVENT_CATEGORIES, + normalizeLifecycleEvent, + readLifecycleEvents, +} = require("../../src/evidence/events"); +const { verifyWorkflowRun } = require("../../src/evidence/verify"); + +const cewpCli = path.join(__dirname, "..", "..", "bin", "cewp.js"); + +function digest(content) { + return `sha256:${crypto.createHash("sha256").update(content).digest("hex")}`; +} + +function runContract() { + const repoRoot = makeTempRepo("cewp-event-run-verify-"); + try { + const approved = approveWorkflow(repoRoot, validDefinition()); + const categories = new Set(Object.values(EVENT_CATEGORIES)); + for (const required of [ + "run", "plan-revision", "task", "checkpoint", "dispatch", "operator-intervention", "verification", + "usage-observation", "estimate-revision", "budget-approval", "allocation-consumption", "threshold", + "warning-presentation", "safe-pause", "unverified-pause", "host-limit", "scope", "review", + "cancellation", "finalize", + ]) assert(categories.has(required), `event vocabulary includes ${required}`); + let found = loadWorkflowRun(repoRoot, approved.runId); + const eventsPath = path.join(found.runRoot, "events.jsonl"); + const events = readLifecycleEvents(eventsPath, { runId: approved.runId }); + assert(events.length === 1, "approved workflow writes one initial lifecycle event"); + assert(events[0].schemaVersion === EVENT_SCHEMA_VERSION, "new lifecycle events use event/v1"); + assert(events[0].category === "run", "workflow approval is categorized as a run event"); + + const legacy = normalizeLifecycleEvent({ + schemaVersion: "workflow-event/v1", + timestamp: "2026-07-22T10:00:00.000Z", + type: "task-started", + runId: approved.runId, + taskId: "implement-example", + }, { allowLegacy: true }); + assert(legacy.schemaVersion === EVENT_SCHEMA_VERSION && legacy.category === "task", "legacy workflow events normalize without rewriting history"); + + const startedAt = new Date(new Date(found.run.createdAt).getTime() + 1000); + startWorkflowTask(found, "implement-example", { now: startedAt }); + found = loadWorkflowRun(repoRoot, approved.runId); + writeEvidenceReceipt(found, { generatedAt: "2026-07-22T10:01:00.000Z" }); + const healthy = verifyWorkflowRun(repoRoot, approved.runId); + assert(healthy.schemaVersion === "run-verification/v1", "run verification is versioned"); + assert(healthy.status === "passed", "consistent run with intact partial receipt passes health verification"); + assert(healthy.execution.agentsExecuted === false && healthy.execution.verificationCommandsExecuted === false, "run verify remains read-only"); + assert(healthy.checks.some((entry) => entry.id === "receipt-integrity" && entry.status === "passed"), "receipt integrity is recomputed"); + + const cli = runNode(cewpCli, ["run", "verify", approved.runId, "--json"], repoRoot); + assert(cli.status === 0, `run verify CLI succeeds for healthy evidence: ${cli.stderr}`); + const cliOutput = JSON.parse(cli.stdout); + assert(cliOutput.schemaVersion === "operator-json/v1" && cliOutput.command === "run.verify", "run verify uses operator JSON"); + + const receiptPath = path.join(found.runRoot, "evidence-receipt.json"); + const receipt = JSON.parse(fs.readFileSync(receiptPath, "utf8")); + receipt.integrity.files[0].sha256 = digest("tampered"); + fs.writeFileSync(receiptPath, `${JSON.stringify(receipt, null, 2)}\n`); + fs.appendFileSync(eventsPath, "{malformed\n"); + fs.appendFileSync(eventsPath, `${JSON.stringify({ + schemaVersion: "event/v999", + category: "task", + timestamp: new Date(startedAt.getTime() + 1000).toISOString(), + type: "task-started", + runId: approved.runId, + })}\n`); + const checkpointDir = path.join(found.runRoot, "checkpoints", "implement-example"); + fs.rmSync(path.join(checkpointDir, fs.readdirSync(checkpointDir)[0])); + writeFile(path.join(found.runRoot, "integration", "host-binding.json"), JSON.stringify({ + schemaVersion: "host-binding/v1", + references: { worktree: { id: "gone", path: path.join(repoRoot, ".cewp-worktrees", "gone") } }, + })); + const unhealthy = verifyWorkflowRun(repoRoot, approved.runId); + assert(unhealthy.status === "failed", "health verification fails closed on evidence defects"); + assert(unhealthy.issues.some((entry) => entry.code === "malformed-event"), "malformed events are reported"); + assert(unhealthy.issues.some((entry) => entry.code === "incompatible-event-schema"), "incompatible event schemas are reported"); + assert(unhealthy.issues.some((entry) => entry.code === "receipt-integrity-mismatch"), "receipt tampering is reported"); + assert(unhealthy.issues.some((entry) => entry.code === "missing-checkpoint"), "missing active checkpoints are reported"); + assert(unhealthy.issues.some((entry) => entry.code === "stale-worktree"), "missing bound worktrees are reported"); + + const failedCli = runNode(cewpCli, ["run", "verify", approved.runId, "--json"], repoRoot); + assert(failedCli.status !== 0, "run verify exits nonzero when health verification fails"); + const failedOutput = JSON.parse(failedCli.stdout); + assert(failedOutput.data.status === "failed", "failed operator JSON remains inspectable"); + } finally { + cleanupRepo(repoRoot); + } +} + +try { + runContract(); + console.log("[PASS] event vocabulary and read-only run verification"); +} catch (error) { + console.error("[FAIL] event vocabulary and run verification contract"); + console.error(error && error.stack ? error.stack : error); + process.exitCode = 1; +} From 0f69cf1cc9d03cd8afe3b5b0bae26ddc4f067224 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tuncay=20=C3=96lmez?= Date: Thu, 23 Jul 2026 13:00:00 +0300 Subject: [PATCH 19/40] Add offline operator reports --- docs/evidence-receipts.md | 11 ++ docs/release-notes.md | 1 + package.json | 5 +- src/cli/parse.js | 2 +- src/cli/usage.js | 1 + src/evidence/receipt.js | 1 + src/evidence/report.js | 168 +++++++++++++++++++++++++++++ src/workflow/cli.js | 15 +++ tests/contracts/operator-report.js | 73 +++++++++++++ 9 files changed, 274 insertions(+), 3 deletions(-) create mode 100644 src/evidence/report.js create mode 100644 tests/contracts/operator-report.js diff --git a/docs/evidence-receipts.md b/docs/evidence-receipts.md index c30d1a6..6456e72 100644 --- a/docs/evidence-receipts.md +++ b/docs/evidence-receipts.md @@ -49,3 +49,14 @@ read-only legacy input and normalized in receipts; CEWP does not rewrite histori required result/checkpoint artifacts, bound-worktree liveness, and every receipt integrity hash. It executes no agent and no approved verification command. A failed check returns a nonzero exit code while `--json` still emits the complete `run-verification/v1` diagnostic model. + +## Offline Operator Report + +`cewp workflow report ` writes `operator-report.json` and a standalone `operator-report.html` in +the workflow run directory. Both are derived from the same normalized receipt model. The HTML uses no +JavaScript, remote fonts, external stylesheets, network requests, server, or control process, so it can be +opened directly from disk on Windows or Linux. + +The report separates observed, estimated, budgeted, and unknown values; shows task progress, revisions, +checkpoint verification, interventions and recovery state, protected reserves, preventive versus observed +controls, and final review. Repository metadata is HTML-escaped, and raw prompts/logs remain excluded. diff --git a/docs/release-notes.md b/docs/release-notes.md index 7e07aab..53854a7 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -8,6 +8,7 @@ - Added `cewp workflow receipt ` without executing agents or verification commands. Raw prompts, transcripts, adapter output, and raw log contents remain excluded by default. - Added the closed `event/v1` lifecycle vocabulary with read-only normalization of historical `workflow-event/v1` records. - Added read-only `cewp run verify ` checks for state, schemas, events, required artifacts, worktree liveness, and receipt integrity. +- Added portable `operator-report/v1` JSON and standalone offline HTML generation from the normalized receipt model. ## 0.11.0-beta.0 diff --git a/package.json b/package.json index 54239bc..b6a48ec 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,7 @@ }, "scripts": { "test": "npm run test:contracts && npm run test:phase12 && npm run smoke", - "test:phase12": "npm run test:evidence-receipt && npm run test:event-run-verify", + "test:phase12": "npm run test:evidence-receipt && npm run test:event-run-verify && npm run test:operator-report", "test:contracts": "npm run test:init-install && npm run test:adapter-profile && npm run test:doctor-json && npm run test:operator-json && npm run test:hook-output && npm run test:skill-format && npm run test:ownership-gates && npm run test:fixtures && npm run test:plugin-package && npm run test:supervised-intake && npm run test:supervised-proposal && npm run test:supervised-controls && npm run test:supervised-execution && npm run test:supervised-review && npm run test:supervised-failure && npm run test:supervised-linear-resume && npm run test:supervised-demo && npm run test:workflow-definition && npm run test:workflow-compiler && npm run test:workflow-proposal && npm run test:workflow-scheduler && npm run test:workflow-worker-matrix && npm run test:workflow-result && npm run test:workflow-failure-result && npm run test:workflow-state-machine && npm run test:workflow-interventions && npm run test:workflow-failure-matrix && npm run test:workflow-budget && npm run test:workflow-progress && npm run test:workflow-review && npm run test:workflow-checkpoint-review && npm run test:workflow-lifecycle && npm run test:workflow-revision && npm run test:workflow-migration && npm run test:workflow-templates && npm run test:workflow-docs && npm run test:workflow-release && npm run test:integration-capabilities && npm run test:integration-binding && npm run test:integration-effort-policy && npm run test:integration-hook-evidence && npm run test:integration-mcp && npm run test:integration-observation && npm run test:native-goal-events", "test:init-install": "node tests/contracts/init-install.js", "test:adapter-profile": "node tests/contracts/adapter-profile.js", @@ -99,6 +99,7 @@ "test:integration-mcp": "node tests/contracts/integration-mcp.js", "test:evidence-receipt": "node tests/contracts/evidence-receipt.js", "test:event-run-verify": "node tests/contracts/event-run-verify.js", + "test:operator-report": "node tests/contracts/operator-report.js", "test:integration-observation": "node tests/contracts/integration-observation.js", "test:native-goal-events": "node tests/contracts/native-goal-events.js", "demo:supervised": "node src/demo/supervised.js", @@ -110,7 +111,7 @@ "check:syntax": "node --check ./bin/cewp.js && node --check ./src/demo/fake-codex.js && node --check ./src/demo/supervised.js && node --check ./src/integration/capabilities.js && node --check ./src/integration/binding.js && node --check ./src/integration/cli.js && node --check ./src/integration/effort-policy.js && node --check ./src/integration/hook-evidence.js && node --check ./src/integration/observation.js && node --check ./src/integration/native-goal.js && node --check ./src/skills/install.js && node --check ./src/skills/format.js && node --check ./src/skills/doctor-report.js && node --check ./src/run/ownership.js && node --check ./src/run/control-gates.js && node --check ./src/supervise/budget.js && node --check ./src/supervise/cli.js && node --check ./src/supervise/commands.js && node --check ./src/supervise/controls.js && node --check ./src/supervise/execution.js && node --check ./src/supervise/profiles.js && node --check ./src/supervise/receipt.js && node --check ./src/supervise/review.js && node --check ./src/supervise/state.js && node --check ./src/supervise/test-authoring.js && node --check ./src/supervise/verification.js && node --check ./src/workflow/cli.js && node --check ./src/workflow/definition.js && node --check ./src/workflow/graph.js && node --check ./src/workflow/result.js && node --check ./src/workflow/scheduler.js && node --check ./src/workflow/source.js && node --check ./src/workflow/state.js && node --check ./src/workflow/transitions.js && node --check ./plugins/cewp/hooks/capture-subagent.js && node --check ./tests/contracts/init-install.js && node --check ./tests/contracts/adapter-profile.js && node --check ./tests/contracts/doctor-json.js && node --check ./tests/contracts/operator-json.js && node --check ./tests/contracts/hook-output.js && node --check ./tests/contracts/skill-format.js && node --check ./tests/contracts/ownership-gates.js && node --check ./tests/contracts/deterministic-fixtures.js && node --check ./tests/contracts/plugin-package.js && node --check ./tests/contracts/supervised-intake.js && node --check ./tests/contracts/supervised-proposal.js && node --check ./tests/contracts/supervised-controls.js && node --check ./tests/contracts/supervised-execution.js && node --check ./tests/contracts/supervised-review.js && node --check ./tests/contracts/supervised-failure.js && node --check ./tests/contracts/supervised-linear-resume.js && node --check ./tests/contracts/supervised-demo.js && node --check ./tests/contracts/integration-capabilities.js && node --check ./tests/contracts/integration-binding.js && node --check ./tests/contracts/integration-effort-policy.js && node --check ./tests/contracts/integration-hook-evidence.js && node --check ./tests/contracts/integration-observation.js && node --check ./tests/contracts/native-goal-events.js && node --check ./tests/contracts/workflow-definition.js && node --check ./tests/contracts/workflow-proposal.js && node --check ./tests/contracts/workflow-scheduler.js && node --check ./tests/contracts/workflow-result.js && node --check ./tests/contracts/workflow-state-machine.js && node --check ./tests/contracts/workflow-interventions.js && node --check ./tests/capabilities/clean-install.js && node --check ./tests/capabilities/plugin-lifecycle.js && node --check ./tests/capabilities/codex-app-server.js && node --check ./tests/capabilities/fixtures/deny-shell-hook.js", "check:workflow-syntax": "node --check ./src/workflow/budget.js && node --check ./src/workflow/compiler.js && node --check ./src/workflow/migration.js && node --check ./src/workflow/progress.js && node --check ./src/workflow/proposal.js && node --check ./src/workflow/review.js && node --check ./src/workflow/revision.js && node --check ./src/workflow/templates.js && node --check ./tests/contracts/workflow-budget.js && node --check ./tests/contracts/workflow-checkpoint-review.js && node --check ./tests/contracts/workflow-compiler.js && node --check ./tests/contracts/workflow-docs.js && node --check ./tests/contracts/workflow-failure-matrix.js && node --check ./tests/contracts/workflow-failure-result.js && node --check ./tests/contracts/workflow-migration.js && node --check ./tests/contracts/workflow-progress.js && node --check ./tests/contracts/workflow-release.js && node --check ./tests/contracts/workflow-review.js && node --check ./tests/contracts/workflow-lifecycle.js && node --check ./tests/contracts/workflow-revision.js && node --check ./tests/contracts/workflow-templates.js && node --check ./tests/contracts/workflow-worker-matrix.js", "check:mcp": "node --check ./bin/cewp-mcp.js && node --check ./src/mcp/server.js && node --check ./src/mcp/tools.js && node --check ./tests/contracts/integration-mcp.js", - "check:evidence": "node --check ./src/evidence/receipt.js && node --check ./src/evidence/events.js && node --check ./src/evidence/verify.js && node --check ./src/run/verify.js && node --check ./tests/contracts/evidence-receipt.js && node --check ./tests/contracts/event-run-verify.js", + "check:evidence": "node --check ./src/evidence/receipt.js && node --check ./src/evidence/events.js && node --check ./src/evidence/report.js && node --check ./src/evidence/verify.js && node --check ./src/run/verify.js && node --check ./tests/contracts/evidence-receipt.js && node --check ./tests/contracts/event-run-verify.js && node --check ./tests/contracts/operator-report.js", "check": "npm run check:syntax && npm run check:workflow-syntax && npm run check:mcp && npm run check:evidence && npm test", "pack:dry-run": "npm pack --dry-run" }, diff --git a/src/cli/parse.js b/src/cli/parse.js index 872ae2c..47a1334 100644 --- a/src/cli/parse.js +++ b/src/cli/parse.js @@ -157,7 +157,7 @@ function parseArgs(argv) { continue; } - if (args.command === "workflow" && ["status", "start", "result", "review", "finalize", "receipt", "intervene", "revise", "apply-revision", "migrate"].includes(args.subcommand) && index === 2 && !arg.startsWith("--")) { + if (args.command === "workflow" && ["status", "start", "result", "review", "finalize", "receipt", "report", "intervene", "revise", "apply-revision", "migrate"].includes(args.subcommand) && index === 2 && !arg.startsWith("--")) { args.workflowRunId = arg; continue; } diff --git a/src/cli/usage.js b/src/cli/usage.js index 7881e0e..5069915 100644 --- a/src/cli/usage.js +++ b/src/cli/usage.js @@ -18,6 +18,7 @@ Usage: cewp workflow review --result --yes [--json] cewp workflow finalize --yes [--json] cewp workflow receipt [--json] + cewp workflow report [--json] cewp integration hooks approve --yes [--json] cewp integration hooks status [--json] cewp integration controls [--json] diff --git a/src/evidence/receipt.js b/src/evidence/receipt.js index d6d35f5..c67d183 100644 --- a/src/evidence/receipt.js +++ b/src/evidence/receipt.js @@ -179,6 +179,7 @@ function checkpointReceipt(entry) { interventionState: entry.value.interventionState, reviewer: entry.value.reviewer, result: entry.value.result, + verification: entry.value.verification, }; } diff --git a/src/evidence/report.js b/src/evidence/report.js new file mode 100644 index 0000000..436c475 --- /dev/null +++ b/src/evidence/report.js @@ -0,0 +1,168 @@ +"use strict"; + +const fs = require("node:fs"); +const path = require("node:path"); +const { writeJsonAtomic } = require("../workflow/state"); +const { buildEvidenceReceipt } = require("./receipt"); + +const OPERATOR_REPORT_SCHEMA_VERSION = "operator-report/v1"; +const ACTIVE_TASK_STATUSES = new Set(["running", "verifying", "review-pending"]); + +function remainingAllocation(entry) { + return Math.max(0, entry.approved - entry.consumed); +} + +function groupControls(receipt) { + const controls = receipt.policy.controls && Array.isArray(receipt.policy.controls.controls) + ? receipt.policy.controls.controls + : []; + return { + enforced: controls.filter((entry) => entry.classification === "preventive"), + checked: controls.filter((entry) => entry.classification === "post-execution"), + observed: controls.filter((entry) => entry.classification === "imported"), + unavailable: controls.filter((entry) => entry.classification === "unavailable"), + claims: receipt.policy.controls ? receipt.policy.controls.claims : { + observedEvidenceIsPreventiveEnforcement: false, + providerExecutionSuppliesEnforcement: false, + preventiveControlAuthority: "none", + guardrailAuthority: "cewp-core-outside-provider-execution", + }, + }; +} + +function buildOperatorReport(receipt) { + if (!receipt || receipt.schemaVersion !== "evidence-receipt/v1") { + throw new Error("Operator report requires evidence-receipt/v1."); + } + const allocationEntries = receipt.budget.compliance.allocations; + const completedTasks = receipt.tasks.filter((task) => task.status === "completed").length; + const activeTasks = receipt.tasks.filter((task) => ACTIVE_TASK_STATUSES.has(task.status)).length; + return { + schemaVersion: OPERATOR_REPORT_SCHEMA_VERSION, + generatedAt: receipt.generatedAt, + runId: receipt.runId, + goal: receipt.goal, + completeness: receipt.completeness, + execution: receipt.execution, + progress: { + runStatus: receipt.completeness.runStatus, + totalTasks: receipt.tasks.length, + completedTasks, + activeTasks, + remainingTasks: receipt.tasks.length - completedTasks, + }, + planRevisions: receipt.planRevisions, + tasks: receipt.tasks, + checkpoints: receipt.checkpoints, + interventions: receipt.interventions, + values: { + observed: receipt.usage, + estimated: receipt.estimate, + budgeted: receipt.budget, + apiEquivalentCost: receipt.cost.apiEquivalent, + }, + reserves: { + absoluteCeilingRespected: receipt.budget.compliance.absoluteCeilingRespected, + protectedAllocationsRespected: receipt.budget.compliance.protectedAllocationsRespected, + allocations: allocationEntries.map((entry) => ({ ...entry, remaining: remainingAllocation(entry) })), + }, + pauseAndRecovery: { + pauseReason: receipt.budget.pauseReason, + resumeStatus: receipt.budget.resumeStatus, + thresholdEvents: receipt.budget.thresholdEvents, + interventions: receipt.interventions, + failures: receipt.tasks.flatMap((task) => task.failure ? [{ taskId: task.id, ...task.failure }] : []), + }, + controls: groupControls(receipt), + finalReview: receipt.reviewer, + reviews: receipt.reviews, + warnings: receipt.warnings, + integrity: { + claim: receipt.integrity.claim, + tamperProof: receipt.integrity.tamperProof, + fileCount: receipt.integrity.files.length, + }, + }; +} + +function escapeHtml(value) { + return String(value === null || value === undefined ? "unknown" : value) + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """) + .replace(/'/g, "'"); +} + +function truthText(value) { + if (!value || value.label === "unknown") return "unknown"; + return `${value.label}: ${value.value}`; +} + +function list(items, render, empty = "None") { + return items.length > 0 + ? `
    ${items.map((item) => `
  • ${render(item)}
    • `).join("")}
` + : `

${escapeHtml(empty)}

`; +} + +function renderOperatorReportHtml(report) { + if (!report || report.schemaVersion !== OPERATOR_REPORT_SCHEMA_VERSION) { + throw new Error(`Operator HTML requires ${OPERATOR_REPORT_SCHEMA_VERSION}.`); + } + const allocationRows = report.reserves.allocations.map((entry) => `${escapeHtml(entry.name)}${entry.approved}${entry.consumed}${entry.remaining}${entry.protected ? "yes" : "no"}${entry.respected ? "passed" : "failed"}`).join(""); + const taskRows = report.tasks.map((task) => `${escapeHtml(task.id)}${escapeHtml(task.status)}${task.attempts}${escapeHtml(task.scopeVerdict.status)}${escapeHtml(task.changedFiles.join(", ") || "none")}`).join(""); + const controlItems = (label, entries) => `

${label}

${list(entries, (entry) => `${escapeHtml(entry.name)} (${escapeHtml(entry.effect)})`)}`; + return ` + + + + + +CEWP Operator Report - ${escapeHtml(report.runId)} + + + +

CEWP Operator Report

+

Run: ${escapeHtml(report.runId)}
Goal: ${escapeHtml(report.goal)}
Status: ${escapeHtml(report.progress.runStatus)} (${escapeHtml(report.completeness.status)})
Execution: ${escapeHtml(report.execution.owner)} / ${escapeHtml(report.execution.backend)}

+

Progress

Tasks: ${report.progress.completedTasks}/${report.progress.totalTasks} complete
Active: ${report.progress.activeTasks}
Remaining: ${report.progress.remainingTasks}
${taskRows}
TaskStatusAttemptsScopeChanged files
+

Plan revisions

${list(report.planRevisions, (entry) => `Revision ${escapeHtml(entry.revision)}: ${escapeHtml(entry.reason || entry.digest)}`)}
+

Checkpoint evidence

${list(report.checkpoints, (entry) => `${escapeHtml(entry.checkpointId)}: ${escapeHtml(entry.status)}; task ${escapeHtml(entry.taskId)}; baseline ${escapeHtml(entry.verification && entry.verification.baseline && entry.verification.baseline.status)}`)}
+

Interventions and recovery

${list(report.interventions, (entry) => `${escapeHtml(entry.event)}: ${escapeHtml(entry.reason)}`)}

Pause: ${escapeHtml(report.pauseAndRecovery.pauseReason)}; resume state: ${escapeHtml(report.pauseAndRecovery.resumeStatus)}

+

Observed, estimated, budgeted, and unknown

Observed managed operations: ${escapeHtml(truthText(report.values.observed.managedOperations))}
Observed host usage: ${escapeHtml(truthText(report.values.observed.hostInternal))}
Estimate: ${escapeHtml(report.values.estimated.label)} (${escapeHtml(report.values.estimated.confidence)})
API-equivalent cost: ${escapeHtml(report.values.apiEquivalentCost.label)}
+

Protected reserves

Absolute ceiling: ${report.reserves.absoluteCeilingRespected ? "passed" : "failed"}; protected allocations: ${report.reserves.protectedAllocationsRespected ? "passed" : "failed"}

${allocationRows}
AllocationApprovedConsumedRemainingProtectedVerdict
+

Controls

${controlItems("Preventively enforced", report.controls.enforced)}${controlItems("Post-execution checked", report.controls.checked)}${controlItems("Observed, not enforced", report.controls.observed)}${controlItems("Unavailable", report.controls.unavailable)}
+

Final review

Status: ${escapeHtml(report.finalReview.status)}; decision: ${escapeHtml(report.finalReview.decision)}

+

Warnings

${list(report.warnings, (entry) => `${escapeHtml(entry.code)}: ${escapeHtml(entry.message)}`)}
+

Generated ${escapeHtml(report.generatedAt)}. Integrity: ${escapeHtml(report.integrity.claim)}; not tamper-proof. This offline report excludes raw prompts and logs.

+ + +`; +} + +function writeTextAtomic(filePath, content) { + const temporaryPath = `${filePath}.${process.pid}.tmp`; + fs.writeFileSync(temporaryPath, content, { flag: "wx" }); + try { + fs.renameSync(temporaryPath, filePath); + } catch (error) { + fs.rmSync(temporaryPath, { force: true }); + throw error; + } +} + +function writeOperatorReport(found, options = {}) { + const receipt = buildEvidenceReceipt(found, options); + const report = buildOperatorReport(receipt); + const jsonPath = path.join(found.runRoot, "operator-report.json"); + const htmlPath = path.join(found.runRoot, "operator-report.html"); + writeJsonAtomic(jsonPath, report); + writeTextAtomic(htmlPath, renderOperatorReportHtml(report)); + return { report, paths: { json: jsonPath, html: htmlPath } }; +} + +module.exports = { + OPERATOR_REPORT_SCHEMA_VERSION, + buildOperatorReport, + renderOperatorReportHtml, + writeOperatorReport, +}; diff --git a/src/workflow/cli.js b/src/workflow/cli.js index 583ac0a..0d73aff 100644 --- a/src/workflow/cli.js +++ b/src/workflow/cli.js @@ -31,6 +31,7 @@ const { const { deriveSchedule } = require("./scheduler"); const { listWorkflowTemplates, loadWorkflowTemplate } = require("./templates"); const { writeEvidenceReceipt } = require("../evidence/receipt"); +const { writeOperatorReport } = require("../evidence/report"); function outputJson(command, data) { console.log(JSON.stringify({ @@ -62,6 +63,20 @@ function resolveProposalSource(options) { } function runWorkflow(options = {}) { + if (options.subcommand === "report") { + if (!options.workflowRunId) throw new Error("workflow report requires a run id."); + const found = loadWorkflowRun(process.cwd(), options.workflowRunId); + const result = writeOperatorReport(found); + if (options.json) outputJson("workflow.report", result); + else { + console.log("CEWP offline operator report written"); + console.log(`Run ID: ${result.report.runId}`); + console.log(`Status: ${result.report.completeness.status}`); + console.log(`JSON: ${result.paths.json}`); + console.log(`HTML: ${result.paths.html}`); + } + return; + } if (options.subcommand === "receipt") { if (!options.workflowRunId) throw new Error("workflow receipt requires a run id."); const found = loadWorkflowRun(process.cwd(), options.workflowRunId); diff --git a/tests/contracts/operator-report.js b/tests/contracts/operator-report.js new file mode 100644 index 0000000..d7b9a97 --- /dev/null +++ b/tests/contracts/operator-report.js @@ -0,0 +1,73 @@ +"use strict"; + +const fs = require("node:fs"); +const path = require("node:path"); +const { assert } = require("../harness/lib/assertions"); +const { cleanupRepo, makeTempRepo, runNode, writeFile } = require("../harness/lib/temp-repo"); +const { validDefinition } = require("./workflow-definition"); +const { approveWorkflow } = require("./workflow-scheduler"); +const { loadWorkflowRun, startWorkflowTask } = require("../../src/workflow/state"); +const { buildEvidenceReceipt } = require("../../src/evidence/receipt"); +const { + buildOperatorReport, + renderOperatorReportHtml, + writeOperatorReport, +} = require("../../src/evidence/report"); + +const cewpCli = path.join(__dirname, "..", "..", "bin", "cewp.js"); + +function runContract() { + const repoRoot = makeTempRepo("cewp-operator-report-"); + try { + const definition = validDefinition(); + definition.goal = "Audit safely"; + const approved = approveWorkflow(repoRoot, definition); + let found = loadWorkflowRun(repoRoot, approved.runId); + startWorkflowTask(found, "implement-example", { + now: new Date(new Date(found.run.createdAt).getTime() + 1000), + }); + found = loadWorkflowRun(repoRoot, approved.runId); + writeFile(path.join(found.runRoot, "adapter-output", "prompt.md"), "REPORT_SECRET_PROMPT\n"); + const generatedAt = "2026-07-22T12:00:00.000Z"; + const receipt = buildEvidenceReceipt(found, { generatedAt }); + const first = buildOperatorReport(receipt); + const second = buildOperatorReport(receipt); + assert(first.schemaVersion === "operator-report/v1", "operator report is versioned"); + assert(JSON.stringify(first) === JSON.stringify(second), "operator report model is deterministic for one receipt"); + assert(first.progress.totalTasks === 2 && first.progress.activeTasks === 1, "report derives task progress"); + assert(Array.isArray(first.planRevisions) && Array.isArray(first.checkpoints), "report exposes revision and checkpoint evidence"); + assert(first.checkpoints[0].verification.baseline.status === "pending", "checkpoint verification evidence is preserved in the report model"); + assert(first.values.observed.managedOperations.label === "unknown", "report preserves observed-versus-unknown truth"); + assert(first.values.estimated.schemaVersion === "usage-estimate/v1", "report exposes estimates separately"); + assert(first.values.budgeted.compliance.status === "passed", "report exposes budget compliance"); + assert(first.reserves.allocations.some((entry) => entry.protected), "report identifies protected reserves"); + assert(first.controls.enforced.length === 0 && first.controls.observed.length === 0, "absent controls are not invented"); + assert(first.finalReview.status === "pending", "report presents final-review state without claiming PASS"); + + const html = renderOperatorReportHtml(first); + assert(html.includes("") && html.includes("Content-Security-Policy"), "report is standalone HTML with an offline policy"); + assert(!html.includes(""), "goal content is HTML escaped"); + assert(html.includes("<script>alert('unsafe')</script>"), "escaped goal remains understandable"); + assert(!/https?:\/\//.test(html), "report has no network dependency"); + assert(!html.includes("REPORT_SECRET_PROMPT"), "report excludes raw prompt content"); + assert(html.includes("Protected reserves") && html.includes("Final review"), "offline report includes critical operator sections"); + + const written = writeOperatorReport(found, { generatedAt }); + assert(fs.existsSync(written.paths.json) && fs.existsSync(written.paths.html), "operator JSON and HTML are written locally"); + const cli = runNode(cewpCli, ["workflow", "report", approved.runId, "--json"], repoRoot); + assert(cli.status === 0, `workflow report CLI succeeds: ${cli.stderr}`); + const output = JSON.parse(cli.stdout); + assert(output.command === "workflow.report" && output.data.report.schemaVersion === "operator-report/v1", "CLI returns the shared report model"); + } finally { + cleanupRepo(repoRoot); + } +} + +try { + runContract(); + console.log("[PASS] static offline operator report"); +} catch (error) { + console.error("[FAIL] operator report contract"); + console.error(error && error.stack ? error.stack : error); + process.exitCode = 1; +} From 2b861129ea76c29698079632431e94594dc42a3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tuncay=20=C3=96lmez?= Date: Thu, 23 Jul 2026 13:30:00 +0300 Subject: [PATCH 20/40] Add honest workflow comparisons --- docs/evidence-receipts.md | 11 +++ docs/release-notes.md | 1 + package.json | 5 +- src/cli/parse.js | 11 +++ src/cli/usage.js | 1 + src/evidence/compare.js | 128 ++++++++++++++++++++++++++++++ src/evidence/receipt.js | 13 ++- src/workflow/cli.js | 19 ++++- tests/contracts/run-comparison.js | 80 +++++++++++++++++++ 9 files changed, 265 insertions(+), 4 deletions(-) create mode 100644 src/evidence/compare.js create mode 100644 tests/contracts/run-comparison.js diff --git a/docs/evidence-receipts.md b/docs/evidence-receipts.md index 6456e72..3fe9518 100644 --- a/docs/evidence-receipts.md +++ b/docs/evidence-receipts.md @@ -60,3 +60,14 @@ opened directly from disk on Windows or Linux. The report separates observed, estimated, budgeted, and unknown values; shows task progress, revisions, checkpoint verification, interventions and recovery state, protected reserves, preventive versus observed controls, and final review. Repository metadata is HTML-escaped, and raw prompts/logs remain excluded. + +## Run Comparison + +`cewp workflow compare ` derives `run-comparison/v1` from two receipts. It +compares outcome, bounded duration, execution owner/backend, observed usage categories, estimates and API +cost when valid, attempts, interventions, failures, scope, commands, and verification evidence. Model time, +CEWP overhead, estimate accuracy, billing cost, or usage that was not observed remains `unknown` and is +excluded from numeric deltas. + +A native-owned workflow is labeled as a native-goal baseline only when a validated host binding includes a +native goal reference. Native ownership alone is insufficient, and unavailable native usage is never zero. diff --git a/docs/release-notes.md b/docs/release-notes.md index 53854a7..c303018 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -9,6 +9,7 @@ - Added the closed `event/v1` lifecycle vocabulary with read-only normalization of historical `workflow-event/v1` records. - Added read-only `cewp run verify ` checks for state, schemas, events, required artifacts, worktree liveness, and receipt integrity. - Added portable `operator-report/v1` JSON and standalone offline HTML generation from the normalized receipt model. +- Added `run-comparison/v1` and `cewp workflow compare` with explicit unknowns and evidence-backed native-goal baseline labeling. ## 0.11.0-beta.0 diff --git a/package.json b/package.json index b6a48ec..7e15f27 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,7 @@ }, "scripts": { "test": "npm run test:contracts && npm run test:phase12 && npm run smoke", - "test:phase12": "npm run test:evidence-receipt && npm run test:event-run-verify && npm run test:operator-report", + "test:phase12": "npm run test:evidence-receipt && npm run test:event-run-verify && npm run test:operator-report && npm run test:run-comparison", "test:contracts": "npm run test:init-install && npm run test:adapter-profile && npm run test:doctor-json && npm run test:operator-json && npm run test:hook-output && npm run test:skill-format && npm run test:ownership-gates && npm run test:fixtures && npm run test:plugin-package && npm run test:supervised-intake && npm run test:supervised-proposal && npm run test:supervised-controls && npm run test:supervised-execution && npm run test:supervised-review && npm run test:supervised-failure && npm run test:supervised-linear-resume && npm run test:supervised-demo && npm run test:workflow-definition && npm run test:workflow-compiler && npm run test:workflow-proposal && npm run test:workflow-scheduler && npm run test:workflow-worker-matrix && npm run test:workflow-result && npm run test:workflow-failure-result && npm run test:workflow-state-machine && npm run test:workflow-interventions && npm run test:workflow-failure-matrix && npm run test:workflow-budget && npm run test:workflow-progress && npm run test:workflow-review && npm run test:workflow-checkpoint-review && npm run test:workflow-lifecycle && npm run test:workflow-revision && npm run test:workflow-migration && npm run test:workflow-templates && npm run test:workflow-docs && npm run test:workflow-release && npm run test:integration-capabilities && npm run test:integration-binding && npm run test:integration-effort-policy && npm run test:integration-hook-evidence && npm run test:integration-mcp && npm run test:integration-observation && npm run test:native-goal-events", "test:init-install": "node tests/contracts/init-install.js", "test:adapter-profile": "node tests/contracts/adapter-profile.js", @@ -100,6 +100,7 @@ "test:evidence-receipt": "node tests/contracts/evidence-receipt.js", "test:event-run-verify": "node tests/contracts/event-run-verify.js", "test:operator-report": "node tests/contracts/operator-report.js", + "test:run-comparison": "node tests/contracts/run-comparison.js", "test:integration-observation": "node tests/contracts/integration-observation.js", "test:native-goal-events": "node tests/contracts/native-goal-events.js", "demo:supervised": "node src/demo/supervised.js", @@ -111,7 +112,7 @@ "check:syntax": "node --check ./bin/cewp.js && node --check ./src/demo/fake-codex.js && node --check ./src/demo/supervised.js && node --check ./src/integration/capabilities.js && node --check ./src/integration/binding.js && node --check ./src/integration/cli.js && node --check ./src/integration/effort-policy.js && node --check ./src/integration/hook-evidence.js && node --check ./src/integration/observation.js && node --check ./src/integration/native-goal.js && node --check ./src/skills/install.js && node --check ./src/skills/format.js && node --check ./src/skills/doctor-report.js && node --check ./src/run/ownership.js && node --check ./src/run/control-gates.js && node --check ./src/supervise/budget.js && node --check ./src/supervise/cli.js && node --check ./src/supervise/commands.js && node --check ./src/supervise/controls.js && node --check ./src/supervise/execution.js && node --check ./src/supervise/profiles.js && node --check ./src/supervise/receipt.js && node --check ./src/supervise/review.js && node --check ./src/supervise/state.js && node --check ./src/supervise/test-authoring.js && node --check ./src/supervise/verification.js && node --check ./src/workflow/cli.js && node --check ./src/workflow/definition.js && node --check ./src/workflow/graph.js && node --check ./src/workflow/result.js && node --check ./src/workflow/scheduler.js && node --check ./src/workflow/source.js && node --check ./src/workflow/state.js && node --check ./src/workflow/transitions.js && node --check ./plugins/cewp/hooks/capture-subagent.js && node --check ./tests/contracts/init-install.js && node --check ./tests/contracts/adapter-profile.js && node --check ./tests/contracts/doctor-json.js && node --check ./tests/contracts/operator-json.js && node --check ./tests/contracts/hook-output.js && node --check ./tests/contracts/skill-format.js && node --check ./tests/contracts/ownership-gates.js && node --check ./tests/contracts/deterministic-fixtures.js && node --check ./tests/contracts/plugin-package.js && node --check ./tests/contracts/supervised-intake.js && node --check ./tests/contracts/supervised-proposal.js && node --check ./tests/contracts/supervised-controls.js && node --check ./tests/contracts/supervised-execution.js && node --check ./tests/contracts/supervised-review.js && node --check ./tests/contracts/supervised-failure.js && node --check ./tests/contracts/supervised-linear-resume.js && node --check ./tests/contracts/supervised-demo.js && node --check ./tests/contracts/integration-capabilities.js && node --check ./tests/contracts/integration-binding.js && node --check ./tests/contracts/integration-effort-policy.js && node --check ./tests/contracts/integration-hook-evidence.js && node --check ./tests/contracts/integration-observation.js && node --check ./tests/contracts/native-goal-events.js && node --check ./tests/contracts/workflow-definition.js && node --check ./tests/contracts/workflow-proposal.js && node --check ./tests/contracts/workflow-scheduler.js && node --check ./tests/contracts/workflow-result.js && node --check ./tests/contracts/workflow-state-machine.js && node --check ./tests/contracts/workflow-interventions.js && node --check ./tests/capabilities/clean-install.js && node --check ./tests/capabilities/plugin-lifecycle.js && node --check ./tests/capabilities/codex-app-server.js && node --check ./tests/capabilities/fixtures/deny-shell-hook.js", "check:workflow-syntax": "node --check ./src/workflow/budget.js && node --check ./src/workflow/compiler.js && node --check ./src/workflow/migration.js && node --check ./src/workflow/progress.js && node --check ./src/workflow/proposal.js && node --check ./src/workflow/review.js && node --check ./src/workflow/revision.js && node --check ./src/workflow/templates.js && node --check ./tests/contracts/workflow-budget.js && node --check ./tests/contracts/workflow-checkpoint-review.js && node --check ./tests/contracts/workflow-compiler.js && node --check ./tests/contracts/workflow-docs.js && node --check ./tests/contracts/workflow-failure-matrix.js && node --check ./tests/contracts/workflow-failure-result.js && node --check ./tests/contracts/workflow-migration.js && node --check ./tests/contracts/workflow-progress.js && node --check ./tests/contracts/workflow-release.js && node --check ./tests/contracts/workflow-review.js && node --check ./tests/contracts/workflow-lifecycle.js && node --check ./tests/contracts/workflow-revision.js && node --check ./tests/contracts/workflow-templates.js && node --check ./tests/contracts/workflow-worker-matrix.js", "check:mcp": "node --check ./bin/cewp-mcp.js && node --check ./src/mcp/server.js && node --check ./src/mcp/tools.js && node --check ./tests/contracts/integration-mcp.js", - "check:evidence": "node --check ./src/evidence/receipt.js && node --check ./src/evidence/events.js && node --check ./src/evidence/report.js && node --check ./src/evidence/verify.js && node --check ./src/run/verify.js && node --check ./tests/contracts/evidence-receipt.js && node --check ./tests/contracts/event-run-verify.js && node --check ./tests/contracts/operator-report.js", + "check:evidence": "node --check ./src/evidence/receipt.js && node --check ./src/evidence/events.js && node --check ./src/evidence/report.js && node --check ./src/evidence/compare.js && node --check ./src/evidence/verify.js && node --check ./src/run/verify.js && node --check ./tests/contracts/evidence-receipt.js && node --check ./tests/contracts/event-run-verify.js && node --check ./tests/contracts/operator-report.js && node --check ./tests/contracts/run-comparison.js", "check": "npm run check:syntax && npm run check:workflow-syntax && npm run check:mcp && npm run check:evidence && npm test", "pack:dry-run": "npm pack --dry-run" }, diff --git a/src/cli/parse.js b/src/cli/parse.js index 47a1334..93d210c 100644 --- a/src/cli/parse.js +++ b/src/cli/parse.js @@ -78,6 +78,7 @@ function parseArgs(argv) { taskClass: undefined, model: undefined, effort: undefined, + comparisonRunId: undefined, }; if (argv[0] === "--help" || argv[0] === "-h") { @@ -157,6 +158,16 @@ function parseArgs(argv) { continue; } + if (args.command === "workflow" && args.subcommand === "compare" && index === 2 && !arg.startsWith("--")) { + args.workflowRunId = arg; + continue; + } + + if (args.command === "workflow" && args.subcommand === "compare" && index === 3 && !arg.startsWith("--")) { + args.comparisonRunId = arg; + continue; + } + if (args.command === "workflow" && ["status", "start", "result", "review", "finalize", "receipt", "report", "intervene", "revise", "apply-revision", "migrate"].includes(args.subcommand) && index === 2 && !arg.startsWith("--")) { args.workflowRunId = arg; continue; diff --git a/src/cli/usage.js b/src/cli/usage.js index 5069915..20487ae 100644 --- a/src/cli/usage.js +++ b/src/cli/usage.js @@ -19,6 +19,7 @@ Usage: cewp workflow finalize --yes [--json] cewp workflow receipt [--json] cewp workflow report [--json] + cewp workflow compare [--json] cewp integration hooks approve --yes [--json] cewp integration hooks status [--json] cewp integration controls [--json] diff --git a/src/evidence/compare.js b/src/evidence/compare.js new file mode 100644 index 0000000..fac2963 --- /dev/null +++ b/src/evidence/compare.js @@ -0,0 +1,128 @@ +"use strict"; + +const RUN_COMPARISON_SCHEMA_VERSION = "run-comparison/v1"; + +function observed(value, source = "cewp-run-state") { + return { label: "observed", value, source }; +} + +function unknown(reason) { + return { label: "unknown", value: null, reason }; +} + +function duration(receipt) { + const start = Date.parse(receipt.timestamps.createdAt); + const end = Date.parse(receipt.timestamps.finalizedAt); + if (!Number.isFinite(start) || !Number.isFinite(end)) return unknown("run is not finalized with a bounded duration"); + return observed(end - start, "cewp-run-timestamps"); +} + +function commandCount(receipt) { + return receipt.commands.reduce((total, task) => ( + total + + (task.verification.baseline ? 1 : 0) + + task.verification.targeted.length + + task.verification.full.length + ), 0); +} + +function failureSummary(receipt) { + const failures = receipt.tasks.flatMap((task) => task.failure ? [{ taskId: task.id, ...task.failure }] : []); + return { count: failures.length, entries: failures }; +} + +function scopeSummary(receipt) { + return { + passed: receipt.tasks.filter((task) => task.scopeVerdict.status === "passed").length, + unknown: receipt.tasks.filter((task) => task.scopeVerdict.status === "unknown").length, + changedFiles: [...new Set(receipt.tasks.flatMap((task) => task.changedFiles))].sort(), + }; +} + +function verificationSummary(receipt) { + const evidence = receipt.tasks.flatMap((task) => [ + ...(task.verification.baseline.evidence || []), + ...task.verification.targeted, + ...task.verification.full, + ]); + return { + commandCount: commandCount(receipt), + passedEvidence: evidence.filter((entry) => entry.status === "passed").length, + failedEvidence: evidence.filter((entry) => entry.status === "failed").length, + evidence, + }; +} + +function summarizeRun(receipt) { + const nativeGoal = receipt.integration && receipt.integration.nativeGoal; + return { + runId: receipt.runId, + execution: receipt.execution, + nativeGoalBaseline: Boolean(nativeGoal && nativeGoal.status === "known"), + nativeGoalEvidence: nativeGoal || unknown("no supported native goal binding"), + }; +} + +function delta(left, right) { + if (left.label !== "observed" || right.label !== "observed") return unknown("both comparison values must be observed"); + return observed(right.value - left.value, "derived-from-observed-values"); +} + +function compareEvidenceReceipts(left, right) { + if (!left || left.schemaVersion !== "evidence-receipt/v1" || !right || right.schemaVersion !== "evidence-receipt/v1") { + throw new Error("Run comparison requires two evidence-receipt/v1 values."); + } + const leftDuration = duration(left); + const rightDuration = duration(right); + const leftAttempts = observed(left.tasks.reduce((total, task) => total + task.attempts, 0)); + const rightAttempts = observed(right.tasks.reduce((total, task) => total + task.attempts, 0)); + const leftInterventions = observed(left.interventions.length); + const rightInterventions = observed(right.interventions.length); + const leftFailures = failureSummary(left); + const rightFailures = failureSummary(right); + const unavailable = []; + if (leftDuration.label === "unknown" || rightDuration.label === "unknown") unavailable.push("duration"); + unavailable.push("model-time", "estimate-accuracy", "cewp-overhead"); + for (const category of ["managedOperations", "capturedOutputBytes", "managedTokens", "hostInternal"]) { + if (left.usage[category].label === "unknown" || right.usage[category].label === "unknown") unavailable.push(`usage.${category}`); + } + if (left.cost.apiEquivalent.label === "unknown" || right.cost.apiEquivalent.label === "unknown") unavailable.push("api-equivalent-cost"); + return { + schemaVersion: RUN_COMPARISON_SCHEMA_VERSION, + generatedAt: left.generatedAt === right.generatedAt ? left.generatedAt : null, + runs: { left: summarizeRun(left), right: summarizeRun(right) }, + dimensions: { + outcome: { + left: { completeness: left.completeness.status, runStatus: left.completeness.runStatus, reviewerDecision: left.reviewer.decision }, + right: { completeness: right.completeness.status, runStatus: right.completeness.runStatus, reviewerDecision: right.reviewer.decision }, + }, + duration: { left: leftDuration, right: rightDuration, delta: delta(leftDuration, rightDuration) }, + modelTime: { left: unknown("effective model time is unavailable"), right: unknown("effective model time is unavailable") }, + usage: { + managedOperations: { left: left.usage.managedOperations, right: right.usage.managedOperations }, + capturedOutputBytes: { left: left.usage.capturedOutputBytes, right: right.usage.capturedOutputBytes }, + managedTokens: { left: left.usage.managedTokens, right: right.usage.managedTokens }, + hostInternal: { left: left.usage.hostInternal, right: right.usage.hostInternal }, + }, + estimateAccuracy: { left: unknown("no calibrated observed outcome mapping"), right: unknown("no calibrated observed outcome mapping") }, + apiEquivalentCost: { left: left.cost.apiEquivalent, right: right.cost.apiEquivalent }, + cewpOverhead: { left: unknown("CEWP overhead was not separately observed"), right: unknown("CEWP overhead was not separately observed") }, + attempts: { left: leftAttempts, right: rightAttempts, delta: delta(leftAttempts, rightAttempts) }, + interventions: { left: leftInterventions, right: rightInterventions, delta: delta(leftInterventions, rightInterventions) }, + failures: { left: leftFailures, right: rightFailures, delta: observed(rightFailures.count - leftFailures.count, "derived-from-run-state") }, + scope: { left: scopeSummary(left), right: scopeSummary(right) }, + commands: { left: left.commands, right: right.commands }, + verification: { left: verificationSummary(left), right: verificationSummary(right) }, + }, + equivalence: { + status: unavailable.length === 0 ? "complete" : "partial", + unavailable: [...new Set(unavailable)].sort(), + warning: unavailable.length > 0 ? "Unavailable values remain unknown and are excluded from numeric deltas." : null, + }, + }; +} + +module.exports = { + RUN_COMPARISON_SCHEMA_VERSION, + compareEvidenceReceipts, +}; diff --git a/src/evidence/receipt.js b/src/evidence/receipt.js index c67d183..3bb4ba2 100644 --- a/src/evidence/receipt.js +++ b/src/evidence/receipt.js @@ -5,7 +5,7 @@ const fs = require("node:fs"); const path = require("node:path"); const { getGitHeadCommit } = require("../lib/git"); const { normalizeSlashPath } = require("../lib/paths"); -const { loadIntegrationControlReceipt } = require("../integration/binding"); +const { loadHostBinding, loadIntegrationControlReceipt } = require("../integration/binding"); const { parseLifecycleEvents } = require("./events"); const { writeJsonAtomic } = require("../workflow/state"); @@ -230,6 +230,7 @@ function buildEvidenceReceipt(found, options = {}) { } const headCommit = getGitHeadCommit(found.repoRoot); const baseCommit = found.run.git && found.run.git.baseCommit; + const hostBinding = loadHostBinding(found); return { schemaVersion: EVIDENCE_RECEIPT_SCHEMA_VERSION, @@ -281,6 +282,16 @@ function buildEvidenceReceipt(found, options = {}) { assurance: found.run.assurance, controls: loadIntegrationControlReceipt(found), }, + integration: { + nativeGoal: hostBinding && hostBinding.execution.owner === "native" && hostBinding.references.goalId + ? { + status: "known", + goalId: hostBinding.references.goalId, + surface: hostBinding.host.surface, + authenticationBoundary: hostBinding.provenance.authenticationBoundary, + } + : { status: "unknown", goalId: null, reason: "no supported native goal binding" }, + }, integrity: { algorithm: "sha256", claim: "tamper-evident-local-metadata", diff --git a/src/workflow/cli.js b/src/workflow/cli.js index 0d73aff..2a9eee3 100644 --- a/src/workflow/cli.js +++ b/src/workflow/cli.js @@ -30,8 +30,9 @@ const { } = require("./state"); const { deriveSchedule } = require("./scheduler"); const { listWorkflowTemplates, loadWorkflowTemplate } = require("./templates"); -const { writeEvidenceReceipt } = require("../evidence/receipt"); +const { buildEvidenceReceipt, writeEvidenceReceipt } = require("../evidence/receipt"); const { writeOperatorReport } = require("../evidence/report"); +const { compareEvidenceReceipts } = require("../evidence/compare"); function outputJson(command, data) { console.log(JSON.stringify({ @@ -63,6 +64,22 @@ function resolveProposalSource(options) { } function runWorkflow(options = {}) { + if (options.subcommand === "compare") { + if (!options.workflowRunId || !options.comparisonRunId) throw new Error("workflow compare requires two run ids."); + const generatedAt = new Date().toISOString(); + const left = buildEvidenceReceipt(loadWorkflowRun(process.cwd(), options.workflowRunId), { generatedAt }); + const right = buildEvidenceReceipt(loadWorkflowRun(process.cwd(), options.comparisonRunId), { generatedAt }); + const comparison = compareEvidenceReceipts(left, right); + if (options.json) outputJson("workflow.compare", comparison); + else { + console.log("CEWP workflow comparison"); + console.log(`Left: ${comparison.runs.left.runId} (${comparison.runs.left.execution.owner})`); + console.log(`Right: ${comparison.runs.right.runId} (${comparison.runs.right.execution.owner})`); + console.log(`Equivalence: ${comparison.equivalence.status}`); + console.log(`Unavailable: ${comparison.equivalence.unavailable.join(", ") || "none"}`); + } + return; + } if (options.subcommand === "report") { if (!options.workflowRunId) throw new Error("workflow report requires a run id."); const found = loadWorkflowRun(process.cwd(), options.workflowRunId); diff --git a/tests/contracts/run-comparison.js b/tests/contracts/run-comparison.js new file mode 100644 index 0000000..4a2015f --- /dev/null +++ b/tests/contracts/run-comparison.js @@ -0,0 +1,80 @@ +"use strict"; + +const path = require("node:path"); +const { assert } = require("../harness/lib/assertions"); +const { cleanupRepo, makeTempRepo, runNode } = require("../harness/lib/temp-repo"); +const { validDefinition } = require("./workflow-definition"); +const { approveWorkflow } = require("./workflow-scheduler"); +const { loadWorkflowRun } = require("../../src/workflow/state"); +const { buildEvidenceReceipt } = require("../../src/evidence/receipt"); +const { compareEvidenceReceipts } = require("../../src/evidence/compare"); +const { createHostBinding } = require("../../src/integration/binding"); +const { supportedSnapshot } = require("./integration-capabilities"); + +const cewpCli = path.join(__dirname, "..", "..", "bin", "cewp.js"); + +function runContract() { + const repoRoot = makeTempRepo("cewp-run-comparison-"); + try { + const managedDefinition = validDefinition(); + managedDefinition.workflowId = "comparison-managed"; + const nativeDefinition = validDefinition(); + nativeDefinition.workflowId = "comparison-native"; + nativeDefinition.execution = { owner: "native", backend: null, allowedModes: ["supervised"] }; + const managed = approveWorkflow(repoRoot, managedDefinition); + const native = approveWorkflow(repoRoot, nativeDefinition); + const nativeFound = loadWorkflowRun(repoRoot, native.runId); + createHostBinding(nativeFound, { + schemaVersion: "host-binding/v1", + workflow: { runId: native.runId, taskId: null, checkpointId: null }, + execution: { owner: "native", backend: null }, + host: { product: "codex", surface: "chatgpt-desktop", version: null }, + mode: "explicit-intake", + provenance: { + kind: "explicit-intake", + capabilitySchemaVersion: null, + authenticationBoundary: "host-owned", + recordedAt: "2026-07-22T12:59:00.000Z", + }, + references: { goalId: "native-goal-baseline-1", threadId: null, turnId: null, subagents: [], worktree: null }, + controls: { preventive: [], postExecution: [], imported: ["goal-reference"], unavailable: ["host-usage"] }, + }, { capabilities: supportedSnapshot() }); + const generatedAt = "2026-07-22T13:00:00.000Z"; + const left = buildEvidenceReceipt(loadWorkflowRun(repoRoot, managed.runId), { generatedAt }); + const right = buildEvidenceReceipt(loadWorkflowRun(repoRoot, native.runId), { generatedAt }); + const comparison = compareEvidenceReceipts(left, right); + assert(comparison.schemaVersion === "run-comparison/v1", "run comparison is versioned"); + assert(comparison.runs.left.execution.owner === "managed" && comparison.runs.right.execution.owner === "native", "execution owner and backend are compared"); + assert(comparison.runs.right.nativeGoalBaseline === true, "native-owner run is labeled as a native-goal baseline"); + const unboundNative = compareEvidenceReceipts(left, { + ...right, + integration: { nativeGoal: { status: "unknown", goalId: null, reason: "no supported native goal binding" } }, + }); + assert(unboundNative.runs.right.nativeGoalBaseline === false, "native ownership alone does not claim a native-goal baseline"); + assert(comparison.dimensions.outcome.left.completeness === "partial", "outcomes remain explicit"); + assert(comparison.dimensions.duration.left.label === "unknown", "unfinished duration remains unknown"); + assert(comparison.dimensions.modelTime.right.label === "unknown", "unavailable native model time remains unknown"); + assert(comparison.dimensions.usage.managedTokens.right.label === "unknown", "unavailable native usage is not converted to zero"); + assert(comparison.dimensions.apiEquivalentCost.right.label === "unknown", "native subscription work does not invent currency cost"); + assert(comparison.dimensions.cewpOverhead.left.label === "unknown", "unobserved CEWP overhead remains unknown"); + assert(comparison.dimensions.attempts.left.label === "observed" && comparison.dimensions.attempts.left.value === 0, "structural zero attempts are observed, not inferred usage"); + assert(comparison.dimensions.commands.left.length > 0 && comparison.dimensions.verification.left.commandCount > 0, "commands and verification evidence are comparable"); + assert(comparison.equivalence.status === "partial" && comparison.equivalence.unavailable.includes("model-time"), "comparison explains unavailable dimensions"); + + const cli = runNode(cewpCli, ["workflow", "compare", managed.runId, native.runId, "--json"], repoRoot); + assert(cli.status === 0, `workflow compare CLI succeeds: ${cli.stderr}`); + const output = JSON.parse(cli.stdout); + assert(output.command === "workflow.compare" && output.data.schemaVersion === "run-comparison/v1", "CLI exposes the shared comparison model"); + } finally { + cleanupRepo(repoRoot); + } +} + +try { + runContract(); + console.log("[PASS] honest managed and native run comparison"); +} catch (error) { + console.error("[FAIL] run comparison contract"); + console.error(error && error.stack ? error.stack : error); + process.exitCode = 1; +} From 19b0d3709c47ad6b73539d5e1af1504b822a43ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tuncay=20=C3=96lmez?= Date: Fri, 24 Jul 2026 09:00:00 +0300 Subject: [PATCH 21/40] Add redacted evidence exports --- docs/evidence-receipts.md | 12 ++- docs/release-notes.md | 1 + package.json | 5 +- src/cli/parse.js | 2 +- src/cli/usage.js | 1 + src/evidence/receipt.js | 1 + src/evidence/redaction.js | 138 ++++++++++++++++++++++++++ src/evidence/report.js | 2 +- src/workflow/cli.js | 12 +++ tests/contracts/evidence-redaction.js | 77 ++++++++++++++ 10 files changed, 245 insertions(+), 6 deletions(-) create mode 100644 src/evidence/redaction.js create mode 100644 tests/contracts/evidence-redaction.js diff --git a/docs/evidence-receipts.md b/docs/evidence-receipts.md index 3fe9518..a3cde1a 100644 --- a/docs/evidence-receipts.md +++ b/docs/evidence-receipts.md @@ -35,8 +35,16 @@ is not implied. Raw prompts, adapter output, transcripts, and raw log contents are excluded by default. The receipt may contain repository-relative paths, changed filenames, approved commands, bounded failure/review summaries, -and artifact names. Inspect these fields before sharing. Phase 12 privacy work will add export redaction; -until then, keep the receipt local if those metadata fields are sensitive. +artifact names, goal/source metadata, host goal identity, revisions, events, timestamps, and reviewer text. +Inspect these fields before sharing. + +`cewp workflow export ` writes separate `.redacted.json`, `.redacted.md`, and `.redacted.html` +artifacts. It removes recognized credential assignments, authorization values, provider-token shapes, +private-key blocks, URL credentials, sensitive/absolute/traversal paths, and active-content markup. It never +overwrites or implicitly creates the canonical receipt. The export records `redaction-policy/v1`, its +replacement count/classes, and that canonical local evidence is still required for integrity verification. +Pattern redaction reduces accidental disclosure but is not a proof that arbitrary prose contains no secret; +inspect exported metadata before sending it outside the repository boundary. ## Event Ledger And Run Health diff --git a/docs/release-notes.md b/docs/release-notes.md index c303018..48dc4f3 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -10,6 +10,7 @@ - Added read-only `cewp run verify ` checks for state, schemas, events, required artifacts, worktree liveness, and receipt integrity. - Added portable `operator-report/v1` JSON and standalone offline HTML generation from the normalized receipt model. - Added `run-comparison/v1` and `cewp workflow compare` with explicit unknowns and evidence-backed native-goal baseline labeling. +- Added adversarially tested `redaction-policy/v1` exports that preserve canonical local evidence and avoid absolute-path output. ## 0.11.0-beta.0 diff --git a/package.json b/package.json index 7e15f27..006b6c7 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,7 @@ }, "scripts": { "test": "npm run test:contracts && npm run test:phase12 && npm run smoke", - "test:phase12": "npm run test:evidence-receipt && npm run test:event-run-verify && npm run test:operator-report && npm run test:run-comparison", + "test:phase12": "npm run test:evidence-receipt && npm run test:event-run-verify && npm run test:operator-report && npm run test:run-comparison && npm run test:evidence-redaction", "test:contracts": "npm run test:init-install && npm run test:adapter-profile && npm run test:doctor-json && npm run test:operator-json && npm run test:hook-output && npm run test:skill-format && npm run test:ownership-gates && npm run test:fixtures && npm run test:plugin-package && npm run test:supervised-intake && npm run test:supervised-proposal && npm run test:supervised-controls && npm run test:supervised-execution && npm run test:supervised-review && npm run test:supervised-failure && npm run test:supervised-linear-resume && npm run test:supervised-demo && npm run test:workflow-definition && npm run test:workflow-compiler && npm run test:workflow-proposal && npm run test:workflow-scheduler && npm run test:workflow-worker-matrix && npm run test:workflow-result && npm run test:workflow-failure-result && npm run test:workflow-state-machine && npm run test:workflow-interventions && npm run test:workflow-failure-matrix && npm run test:workflow-budget && npm run test:workflow-progress && npm run test:workflow-review && npm run test:workflow-checkpoint-review && npm run test:workflow-lifecycle && npm run test:workflow-revision && npm run test:workflow-migration && npm run test:workflow-templates && npm run test:workflow-docs && npm run test:workflow-release && npm run test:integration-capabilities && npm run test:integration-binding && npm run test:integration-effort-policy && npm run test:integration-hook-evidence && npm run test:integration-mcp && npm run test:integration-observation && npm run test:native-goal-events", "test:init-install": "node tests/contracts/init-install.js", "test:adapter-profile": "node tests/contracts/adapter-profile.js", @@ -101,6 +101,7 @@ "test:event-run-verify": "node tests/contracts/event-run-verify.js", "test:operator-report": "node tests/contracts/operator-report.js", "test:run-comparison": "node tests/contracts/run-comparison.js", + "test:evidence-redaction": "node tests/contracts/evidence-redaction.js", "test:integration-observation": "node tests/contracts/integration-observation.js", "test:native-goal-events": "node tests/contracts/native-goal-events.js", "demo:supervised": "node src/demo/supervised.js", @@ -112,7 +113,7 @@ "check:syntax": "node --check ./bin/cewp.js && node --check ./src/demo/fake-codex.js && node --check ./src/demo/supervised.js && node --check ./src/integration/capabilities.js && node --check ./src/integration/binding.js && node --check ./src/integration/cli.js && node --check ./src/integration/effort-policy.js && node --check ./src/integration/hook-evidence.js && node --check ./src/integration/observation.js && node --check ./src/integration/native-goal.js && node --check ./src/skills/install.js && node --check ./src/skills/format.js && node --check ./src/skills/doctor-report.js && node --check ./src/run/ownership.js && node --check ./src/run/control-gates.js && node --check ./src/supervise/budget.js && node --check ./src/supervise/cli.js && node --check ./src/supervise/commands.js && node --check ./src/supervise/controls.js && node --check ./src/supervise/execution.js && node --check ./src/supervise/profiles.js && node --check ./src/supervise/receipt.js && node --check ./src/supervise/review.js && node --check ./src/supervise/state.js && node --check ./src/supervise/test-authoring.js && node --check ./src/supervise/verification.js && node --check ./src/workflow/cli.js && node --check ./src/workflow/definition.js && node --check ./src/workflow/graph.js && node --check ./src/workflow/result.js && node --check ./src/workflow/scheduler.js && node --check ./src/workflow/source.js && node --check ./src/workflow/state.js && node --check ./src/workflow/transitions.js && node --check ./plugins/cewp/hooks/capture-subagent.js && node --check ./tests/contracts/init-install.js && node --check ./tests/contracts/adapter-profile.js && node --check ./tests/contracts/doctor-json.js && node --check ./tests/contracts/operator-json.js && node --check ./tests/contracts/hook-output.js && node --check ./tests/contracts/skill-format.js && node --check ./tests/contracts/ownership-gates.js && node --check ./tests/contracts/deterministic-fixtures.js && node --check ./tests/contracts/plugin-package.js && node --check ./tests/contracts/supervised-intake.js && node --check ./tests/contracts/supervised-proposal.js && node --check ./tests/contracts/supervised-controls.js && node --check ./tests/contracts/supervised-execution.js && node --check ./tests/contracts/supervised-review.js && node --check ./tests/contracts/supervised-failure.js && node --check ./tests/contracts/supervised-linear-resume.js && node --check ./tests/contracts/supervised-demo.js && node --check ./tests/contracts/integration-capabilities.js && node --check ./tests/contracts/integration-binding.js && node --check ./tests/contracts/integration-effort-policy.js && node --check ./tests/contracts/integration-hook-evidence.js && node --check ./tests/contracts/integration-observation.js && node --check ./tests/contracts/native-goal-events.js && node --check ./tests/contracts/workflow-definition.js && node --check ./tests/contracts/workflow-proposal.js && node --check ./tests/contracts/workflow-scheduler.js && node --check ./tests/contracts/workflow-result.js && node --check ./tests/contracts/workflow-state-machine.js && node --check ./tests/contracts/workflow-interventions.js && node --check ./tests/capabilities/clean-install.js && node --check ./tests/capabilities/plugin-lifecycle.js && node --check ./tests/capabilities/codex-app-server.js && node --check ./tests/capabilities/fixtures/deny-shell-hook.js", "check:workflow-syntax": "node --check ./src/workflow/budget.js && node --check ./src/workflow/compiler.js && node --check ./src/workflow/migration.js && node --check ./src/workflow/progress.js && node --check ./src/workflow/proposal.js && node --check ./src/workflow/review.js && node --check ./src/workflow/revision.js && node --check ./src/workflow/templates.js && node --check ./tests/contracts/workflow-budget.js && node --check ./tests/contracts/workflow-checkpoint-review.js && node --check ./tests/contracts/workflow-compiler.js && node --check ./tests/contracts/workflow-docs.js && node --check ./tests/contracts/workflow-failure-matrix.js && node --check ./tests/contracts/workflow-failure-result.js && node --check ./tests/contracts/workflow-migration.js && node --check ./tests/contracts/workflow-progress.js && node --check ./tests/contracts/workflow-release.js && node --check ./tests/contracts/workflow-review.js && node --check ./tests/contracts/workflow-lifecycle.js && node --check ./tests/contracts/workflow-revision.js && node --check ./tests/contracts/workflow-templates.js && node --check ./tests/contracts/workflow-worker-matrix.js", "check:mcp": "node --check ./bin/cewp-mcp.js && node --check ./src/mcp/server.js && node --check ./src/mcp/tools.js && node --check ./tests/contracts/integration-mcp.js", - "check:evidence": "node --check ./src/evidence/receipt.js && node --check ./src/evidence/events.js && node --check ./src/evidence/report.js && node --check ./src/evidence/compare.js && node --check ./src/evidence/verify.js && node --check ./src/run/verify.js && node --check ./tests/contracts/evidence-receipt.js && node --check ./tests/contracts/event-run-verify.js && node --check ./tests/contracts/operator-report.js && node --check ./tests/contracts/run-comparison.js", + "check:evidence": "node --check ./src/evidence/receipt.js && node --check ./src/evidence/events.js && node --check ./src/evidence/report.js && node --check ./src/evidence/compare.js && node --check ./src/evidence/redaction.js && node --check ./src/evidence/verify.js && node --check ./src/run/verify.js && node --check ./tests/contracts/evidence-receipt.js && node --check ./tests/contracts/event-run-verify.js && node --check ./tests/contracts/operator-report.js && node --check ./tests/contracts/run-comparison.js && node --check ./tests/contracts/evidence-redaction.js", "check": "npm run check:syntax && npm run check:workflow-syntax && npm run check:mcp && npm run check:evidence && npm test", "pack:dry-run": "npm pack --dry-run" }, diff --git a/src/cli/parse.js b/src/cli/parse.js index 93d210c..64a73fc 100644 --- a/src/cli/parse.js +++ b/src/cli/parse.js @@ -168,7 +168,7 @@ function parseArgs(argv) { continue; } - if (args.command === "workflow" && ["status", "start", "result", "review", "finalize", "receipt", "report", "intervene", "revise", "apply-revision", "migrate"].includes(args.subcommand) && index === 2 && !arg.startsWith("--")) { + if (args.command === "workflow" && ["status", "start", "result", "review", "finalize", "receipt", "report", "export", "intervene", "revise", "apply-revision", "migrate"].includes(args.subcommand) && index === 2 && !arg.startsWith("--")) { args.workflowRunId = arg; continue; } diff --git a/src/cli/usage.js b/src/cli/usage.js index 20487ae..8b3eaf6 100644 --- a/src/cli/usage.js +++ b/src/cli/usage.js @@ -20,6 +20,7 @@ Usage: cewp workflow receipt [--json] cewp workflow report [--json] cewp workflow compare [--json] + cewp workflow export [--json] cewp integration hooks approve --yes [--json] cewp integration hooks status [--json] cewp integration controls [--json] diff --git a/src/evidence/receipt.js b/src/evidence/receipt.js index 3bb4ba2..7f54a0d 100644 --- a/src/evidence/receipt.js +++ b/src/evidence/receipt.js @@ -335,6 +335,7 @@ function renderEvidenceReceiptMarkdown(receipt) { - Host-internal usage: ${receipt.usage.hostInternal.label} - API-equivalent cost: ${receipt.cost.apiEquivalent.label} - Integrity: ${receipt.integrity.claim}; ${receipt.integrity.files.length} hashed files; not tamper-proof +${receipt.redaction && receipt.redaction.applied ? `- Redaction: applied (${receipt.redaction.replacements} replacements)\n` : ""} ## Tasks diff --git a/src/evidence/redaction.js b/src/evidence/redaction.js new file mode 100644 index 0000000..ccbe8a9 --- /dev/null +++ b/src/evidence/redaction.js @@ -0,0 +1,138 @@ +"use strict"; + +const fs = require("node:fs"); +const path = require("node:path"); +const { normalizeSlashPath } = require("../lib/paths"); +const { writeJsonAtomic } = require("../workflow/state"); +const { buildEvidenceReceipt, renderEvidenceReceiptMarkdown } = require("./receipt"); +const { buildOperatorReport, renderOperatorReportHtml } = require("./report"); + +const REDACTION_SCHEMA_VERSION = "redaction-policy/v1"; +const SENSITIVE_KEY = /^(authorization|password|passwd|secret|client[_-]?secret|api[_-]?key|access[_-]?token|refresh[_-]?token|session[_-]?token|auth[_-]?token|cookie|private[_-]?key|credential)$/i; +const PATH_KEY = /(^|[A-Z_-])(paths?|files?|director(?:y|ies))$/i; +const SENSITIVE_PATH = /(^|[\\/])(?:\.env(?:\.[^\\/]*)?|id_rsa|id_ed25519|credentials?(?:\.[^\\/]*)?|[^\\/]+\.(?:pem|p12|pfx|key))$/i; +const ABSOLUTE_WINDOWS_PATH = /^[a-z]:[\\/]/i; +const ABSOLUTE_UNC_PATH = /^\\\\/; + +function redactString(input, key, state) { + if (SENSITIVE_KEY.test(key || "")) { + state.replacements += 1; + state.classes.add("sensitive-key"); + return "[REDACTED]"; + } + if (PATH_KEY.test(key || "") && ( + path.isAbsolute(input) + || ABSOLUTE_WINDOWS_PATH.test(input) + || ABSOLUTE_UNC_PATH.test(input) + || /^~[\\/]/.test(input) + || /^(?:file:|\$(?:\{)?HOME|%(?:USERPROFILE|HOME)%)/i.test(input) + || input.split(/[\\/]+/).includes("..") + || SENSITIVE_PATH.test(input) + )) { + state.replacements += 1; + state.classes.add("sensitive-path"); + return "[REDACTED_PATH]"; + } + let value = input; + const patterns = [ + { className: "active-content", pattern: /<\/?(?:script|style|iframe|object|embed|link|meta)\b[^>]*>/gi, replacement: "[REDACTED_MARKUP]" }, + { className: "private-key", pattern: /-----BEGIN [^-\r\n]*PRIVATE KEY-----[\s\S]*?-----END [^-\r\n]*PRIVATE KEY-----/g, replacement: "[REDACTED_PRIVATE_KEY]" }, + { className: "authorization", pattern: /\bBearer\s+[A-Za-z0-9._~+\/-]+=*/gi, replacement: "Bearer [REDACTED]" }, + { className: "url-credentials", pattern: /([a-z][a-z0-9+.-]*:\/\/)[^\s/@:]+:[^\s/@]+@/gi, replacement: "$1[REDACTED]@" }, + { className: "provider-token", pattern: /\b(?:gh[pousr]_[A-Za-z0-9_]{20,}|glpat-[A-Za-z0-9_-]{20,}|npm_[A-Za-z0-9]{20,}|sk-[A-Za-z0-9_-]{20,}|AKIA[A-Z0-9]{16}|xox[baprs]-[A-Za-z0-9-]{20,})\b/g, replacement: "[REDACTED_TOKEN]" }, + { className: "assigned-secret", pattern: /((?:--)?(?:password|passwd|token|secret|api[_-]?key)\s*[=:]\s*)[^\s,;]+/gi, replacement: "$1[REDACTED]" }, + { className: "embedded-path", pattern: /\b[A-Za-z]:[\\/](?:[^\s,;<>"']+[\\/]?)+|\/(?:home|Users|root|tmp|var\/tmp)\/[^\s,;<>"']+/g, replacement: "[REDACTED_PATH]" }, + ]; + for (const entry of patterns) { + let count = 0; + value = value.replace(entry.pattern, (...args) => { + count += 1; + return typeof entry.replacement === "function" ? entry.replacement(...args) : entry.replacement.replace("$1", args[1] || ""); + }); + if (count > 0) { + state.replacements += count; + state.classes.add(entry.className); + } + } + return value; +} + +function redactNode(value, key, state) { + if (typeof value === "string") return redactString(value, key, state); + if (Array.isArray(value)) return value.map((entry) => redactNode(entry, key, state)); + if (!value || typeof value !== "object") return value; + return Object.fromEntries(Object.entries(value).map(([entryKey, entryValue]) => [ + entryKey, + redactNode(entryValue, entryKey, state), + ])); +} + +function redactEvidenceValue(value) { + const state = { replacements: 0, classes: new Set() }; + return { + value: redactNode(value, "", state), + replacements: state.replacements, + classes: [...state.classes].sort(), + }; +} + +function redactEvidenceReceipt(receipt) { + const redacted = redactEvidenceValue(receipt); + return { + ...redacted.value, + redaction: { + schemaVersion: REDACTION_SCHEMA_VERSION, + applied: true, + replacements: redacted.replacements, + classes: redacted.classes, + canonicalReceiptModified: false, + rawPromptsIncluded: false, + rawLogsIncluded: false, + }, + integrity: { + ...redacted.value.integrity, + exportRedaction: { + applied: true, + canonicalLocalReceiptRequiredForVerification: true, + }, + }, + }; +} + +function writeTextAtomic(filePath, content) { + const temporaryPath = `${filePath}.${process.pid}.tmp`; + fs.writeFileSync(temporaryPath, content, { flag: "wx" }); + try { + fs.renameSync(temporaryPath, filePath); + } catch (error) { + fs.rmSync(temporaryPath, { force: true }); + throw error; + } +} + +function exportRedactedEvidence(found, options = {}) { + const receipt = redactEvidenceReceipt(buildEvidenceReceipt(found, options)); + const report = { ...buildOperatorReport(receipt), redaction: receipt.redaction }; + const absolutePaths = { + receiptJson: path.join(found.runRoot, "evidence-receipt.redacted.json"), + receiptMarkdown: path.join(found.runRoot, "evidence-receipt.redacted.md"), + reportJson: path.join(found.runRoot, "operator-report.redacted.json"), + html: path.join(found.runRoot, "operator-report.redacted.html"), + }; + writeJsonAtomic(absolutePaths.receiptJson, receipt); + writeTextAtomic(absolutePaths.receiptMarkdown, renderEvidenceReceiptMarkdown(receipt)); + writeJsonAtomic(absolutePaths.reportJson, report); + writeTextAtomic(absolutePaths.html, renderOperatorReportHtml(report)); + const paths = Object.fromEntries(Object.entries(absolutePaths).map(([name, filePath]) => [ + name, + normalizeSlashPath(path.relative(found.repoRoot, filePath)), + ])); + return { receipt, report, paths }; +} + +module.exports = { + REDACTION_SCHEMA_VERSION, + exportRedactedEvidence, + redactEvidenceReceipt, + redactEvidenceValue, +}; diff --git a/src/evidence/report.js b/src/evidence/report.js index 436c475..b378bea 100644 --- a/src/evidence/report.js +++ b/src/evidence/report.js @@ -133,7 +133,7 @@ function renderOperatorReportHtml(report) {

Controls

${controlItems("Preventively enforced", report.controls.enforced)}${controlItems("Post-execution checked", report.controls.checked)}${controlItems("Observed, not enforced", report.controls.observed)}${controlItems("Unavailable", report.controls.unavailable)}

Final review

Status: ${escapeHtml(report.finalReview.status)}; decision: ${escapeHtml(report.finalReview.decision)}

Warnings

${list(report.warnings, (entry) => `${escapeHtml(entry.code)}: ${escapeHtml(entry.message)}`)}
-

Generated ${escapeHtml(report.generatedAt)}. Integrity: ${escapeHtml(report.integrity.claim)}; not tamper-proof. This offline report excludes raw prompts and logs.

+

Generated ${escapeHtml(report.generatedAt)}. Integrity: ${escapeHtml(report.integrity.claim)}; not tamper-proof. Redaction: ${report.redaction && report.redaction.applied ? `applied (${report.redaction.replacements} replacements)` : "not applied"}. This offline report excludes raw prompts and logs.

`; diff --git a/src/workflow/cli.js b/src/workflow/cli.js index 2a9eee3..442b9aa 100644 --- a/src/workflow/cli.js +++ b/src/workflow/cli.js @@ -33,6 +33,7 @@ const { listWorkflowTemplates, loadWorkflowTemplate } = require("./templates"); const { buildEvidenceReceipt, writeEvidenceReceipt } = require("../evidence/receipt"); const { writeOperatorReport } = require("../evidence/report"); const { compareEvidenceReceipts } = require("../evidence/compare"); +const { exportRedactedEvidence } = require("../evidence/redaction"); function outputJson(command, data) { console.log(JSON.stringify({ @@ -64,6 +65,17 @@ function resolveProposalSource(options) { } function runWorkflow(options = {}) { + if (options.subcommand === "export") { + if (!options.workflowRunId) throw new Error("workflow export requires a run id."); + const result = exportRedactedEvidence(loadWorkflowRun(process.cwd(), options.workflowRunId)); + if (options.json) outputJson("workflow.export", result); + else { + console.log("CEWP redacted evidence export written"); + console.log(`Run ID: ${result.receipt.runId}`); + for (const [name, filePath] of Object.entries(result.paths)) console.log(`${name}: ${filePath}`); + } + return; + } if (options.subcommand === "compare") { if (!options.workflowRunId || !options.comparisonRunId) throw new Error("workflow compare requires two run ids."); const generatedAt = new Date().toISOString(); diff --git a/tests/contracts/evidence-redaction.js b/tests/contracts/evidence-redaction.js new file mode 100644 index 0000000..dc82abc --- /dev/null +++ b/tests/contracts/evidence-redaction.js @@ -0,0 +1,77 @@ +"use strict"; + +const fs = require("node:fs"); +const path = require("node:path"); +const { assert } = require("../harness/lib/assertions"); +const { cleanupRepo, makeTempRepo, runNode, writeFile } = require("../harness/lib/temp-repo"); +const { validDefinition } = require("./workflow-definition"); +const { approveWorkflow } = require("./workflow-scheduler"); +const { loadWorkflowRun } = require("../../src/workflow/state"); +const { exportRedactedEvidence, redactEvidenceValue } = require("../../src/evidence/redaction"); + +const cewpCli = path.join(__dirname, "..", "..", "bin", "cewp.js"); +const SECRET = "ghp_abcdefghijklmnopqrstuvwxyz1234567890"; + +function runContract() { + const synthetic = redactEvidenceValue({ + authorization: "Bearer top-secret-bearer", + password: "hunter2", + clientSecret: "client-secret-value", + command: `tool --token=${SECRET} --count=2`, + url: "https://alice:private@example.invalid/repo", + privateBlock: "-----BEGIN PRIVATE KEY-----\nabc\n-----END PRIVATE KEY-----", + absoluteWindowsPath: "C:\\Users\\Alice\\private\\notes.txt", + traversalPath: "../../etc/passwd", + dotenvPath: "config/.env", + changedFiles: ["src/safe.js", "config/.env"], + note: "See C:\\Users\\Alice\\private\\notes.txt before export", + managedTokens: { label: "observed", value: 42 }, + }); + const serializedSynthetic = JSON.stringify(synthetic.value); + for (const secret of ["top-secret-bearer", "hunter2", "client-secret-value", SECRET, "alice:private", "BEGIN PRIVATE KEY", "Users\\\\Alice", "../../etc/passwd", "config/.env"]) { + assert(!serializedSynthetic.includes(secret), `redaction removes adversarial value ${secret}`); + } + assert(synthetic.value.managedTokens.value === 42, "non-secret usage token counts are preserved"); + assert(synthetic.value.changedFiles[0] === "src/safe.js" && synthetic.value.changedFiles[1] === "[REDACTED_PATH]", "path arrays preserve safe paths and redact sensitive paths"); + assert(synthetic.replacements >= 7, "redaction reports replacement count"); + + const repoRoot = makeTempRepo("cewp-evidence-redaction-"); + try { + const definition = validDefinition(); + definition.workflowId = "redacted-export"; + definition.goal = `Fix using token=${SECRET}`; + definition.tasks[0].verification.targeted[0] = `node test.js --password=hunter2`; + const approved = approveWorkflow(repoRoot, definition); + const found = loadWorkflowRun(repoRoot, approved.runId); + writeFile(path.join(found.runRoot, "adapter-output", "prompt.md"), "RAW_PROMPT_DO_NOT_EXPORT\n"); + const exported = exportRedactedEvidence(found, { generatedAt: "2026-07-22T14:00:00.000Z" }); + assert(exported.receipt.schemaVersion === "evidence-receipt/v1", "redacted export preserves receipt contract"); + assert(exported.receipt.redaction.schemaVersion === "redaction-policy/v1" && exported.receipt.redaction.applied, "redaction is explicit and versioned"); + assert(exported.report.schemaVersion === "operator-report/v1", "redacted export preserves report contract"); + assert(Object.values(exported.paths).every((entry) => !path.isAbsolute(entry) && !entry.includes("..")), "export returns repository-relative paths"); + const contents = Object.values(exported.paths).map((relative) => fs.readFileSync(path.join(repoRoot, relative), "utf8")).join("\n"); + for (const secret of [SECRET, "hunter2", "RAW_PROMPT_DO_NOT_EXPORT", ""]) { + assert(!contents.includes(secret), `export excludes ${secret}`); + } + assert(!/https?:\/\//.test(fs.readFileSync(path.join(repoRoot, exported.paths.html), "utf8")), "redacted HTML remains offline"); + assert(contents.includes("Redaction") && contents.includes("redaction-policy/v1"), "export artifacts disclose that redaction was applied"); + assert(!fs.existsSync(path.join(found.runRoot, "evidence-receipt.json")), "export does not overwrite or create the canonical receipt"); + + const cli = runNode(cewpCli, ["workflow", "export", approved.runId, "--json"], repoRoot); + assert(cli.status === 0, `workflow export CLI succeeds: ${cli.stderr}`); + const output = JSON.parse(cli.stdout); + assert(output.command === "workflow.export" && output.data.receipt.redaction.applied, "CLI emits only the redacted export model"); + assert(!cli.stdout.includes(SECRET) && !cli.stdout.includes(repoRoot), "CLI JSON does not leak secrets or absolute repository paths"); + } finally { + cleanupRepo(repoRoot); + } +} + +try { + runContract(); + console.log("[PASS] adversarial evidence redaction and safe export"); +} catch (error) { + console.error("[FAIL] evidence redaction contract"); + console.error(error && error.stack ? error.stack : error); + process.exitCode = 1; +} From 318dc2082efddf8a47aeb527542ec3e3fbd4681a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tuncay=20=C3=96lmez?= Date: Fri, 24 Jul 2026 09:30:00 +0300 Subject: [PATCH 22/40] Add usage evidence provenance --- docs/evidence-receipts.md | 13 +++ docs/release-notes.md | 1 + package.json | 2 +- src/evidence/receipt.js | 12 ++- src/evidence/usage.js | 139 ++++++++++++++++++++++++++++ src/integration/observation.js | 11 +++ tests/contracts/evidence-receipt.js | 51 ++++++++++ 7 files changed, 226 insertions(+), 3 deletions(-) create mode 100644 src/evidence/usage.js diff --git a/docs/evidence-receipts.md b/docs/evidence-receipts.md index a3cde1a..eaca93c 100644 --- a/docs/evidence-receipts.md +++ b/docs/evidence-receipts.md @@ -79,3 +79,16 @@ excluded from numeric deltas. A native-owned workflow is labeled as a native-goal baseline only when a validated host binding includes a native goal reference. Native ownership alone is insufficient, and unavailable native usage is never zero. + +## Usage And Estimate Truth + +Each task-result, review-result, and supported host usage record becomes a separate +`usage-observation/v1`. The receipt retains its normalized category, raw category name, observed/imported/ +unknown availability, source schema, authentication boundary, timestamp, scope, and effective model only +when known. Bounded raw host payloads are not copied into the receipt. Imported observations stay imported, +do not enter observed totals, and never imply billing impact. + +`usage-estimate/v1` records estimator version/method, grouping dimensions, local sample basis, calibration +snapshot, and drift state. With fewer than five comparable runs—or without known model/effort—it remains an +unknown range with unavailable confidence. CEWP does not promote a numeric estimate from fixtures or +non-comparable history. diff --git a/docs/release-notes.md b/docs/release-notes.md index 48dc4f3..d5cdf8f 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -11,6 +11,7 @@ - Added portable `operator-report/v1` JSON and standalone offline HTML generation from the normalized receipt model. - Added `run-comparison/v1` and `cewp workflow compare` with explicit unknowns and evidence-backed native-goal baseline labeling. - Added adversarially tested `redaction-policy/v1` exports that preserve canonical local evidence and avoid absolute-path output. +- Added `usage-observation/v1` provenance/raw-category records and reproducible unknown `usage-estimate/v1` calibration metadata. ## 0.11.0-beta.0 diff --git a/package.json b/package.json index 006b6c7..d777533 100644 --- a/package.json +++ b/package.json @@ -113,7 +113,7 @@ "check:syntax": "node --check ./bin/cewp.js && node --check ./src/demo/fake-codex.js && node --check ./src/demo/supervised.js && node --check ./src/integration/capabilities.js && node --check ./src/integration/binding.js && node --check ./src/integration/cli.js && node --check ./src/integration/effort-policy.js && node --check ./src/integration/hook-evidence.js && node --check ./src/integration/observation.js && node --check ./src/integration/native-goal.js && node --check ./src/skills/install.js && node --check ./src/skills/format.js && node --check ./src/skills/doctor-report.js && node --check ./src/run/ownership.js && node --check ./src/run/control-gates.js && node --check ./src/supervise/budget.js && node --check ./src/supervise/cli.js && node --check ./src/supervise/commands.js && node --check ./src/supervise/controls.js && node --check ./src/supervise/execution.js && node --check ./src/supervise/profiles.js && node --check ./src/supervise/receipt.js && node --check ./src/supervise/review.js && node --check ./src/supervise/state.js && node --check ./src/supervise/test-authoring.js && node --check ./src/supervise/verification.js && node --check ./src/workflow/cli.js && node --check ./src/workflow/definition.js && node --check ./src/workflow/graph.js && node --check ./src/workflow/result.js && node --check ./src/workflow/scheduler.js && node --check ./src/workflow/source.js && node --check ./src/workflow/state.js && node --check ./src/workflow/transitions.js && node --check ./plugins/cewp/hooks/capture-subagent.js && node --check ./tests/contracts/init-install.js && node --check ./tests/contracts/adapter-profile.js && node --check ./tests/contracts/doctor-json.js && node --check ./tests/contracts/operator-json.js && node --check ./tests/contracts/hook-output.js && node --check ./tests/contracts/skill-format.js && node --check ./tests/contracts/ownership-gates.js && node --check ./tests/contracts/deterministic-fixtures.js && node --check ./tests/contracts/plugin-package.js && node --check ./tests/contracts/supervised-intake.js && node --check ./tests/contracts/supervised-proposal.js && node --check ./tests/contracts/supervised-controls.js && node --check ./tests/contracts/supervised-execution.js && node --check ./tests/contracts/supervised-review.js && node --check ./tests/contracts/supervised-failure.js && node --check ./tests/contracts/supervised-linear-resume.js && node --check ./tests/contracts/supervised-demo.js && node --check ./tests/contracts/integration-capabilities.js && node --check ./tests/contracts/integration-binding.js && node --check ./tests/contracts/integration-effort-policy.js && node --check ./tests/contracts/integration-hook-evidence.js && node --check ./tests/contracts/integration-observation.js && node --check ./tests/contracts/native-goal-events.js && node --check ./tests/contracts/workflow-definition.js && node --check ./tests/contracts/workflow-proposal.js && node --check ./tests/contracts/workflow-scheduler.js && node --check ./tests/contracts/workflow-result.js && node --check ./tests/contracts/workflow-state-machine.js && node --check ./tests/contracts/workflow-interventions.js && node --check ./tests/capabilities/clean-install.js && node --check ./tests/capabilities/plugin-lifecycle.js && node --check ./tests/capabilities/codex-app-server.js && node --check ./tests/capabilities/fixtures/deny-shell-hook.js", "check:workflow-syntax": "node --check ./src/workflow/budget.js && node --check ./src/workflow/compiler.js && node --check ./src/workflow/migration.js && node --check ./src/workflow/progress.js && node --check ./src/workflow/proposal.js && node --check ./src/workflow/review.js && node --check ./src/workflow/revision.js && node --check ./src/workflow/templates.js && node --check ./tests/contracts/workflow-budget.js && node --check ./tests/contracts/workflow-checkpoint-review.js && node --check ./tests/contracts/workflow-compiler.js && node --check ./tests/contracts/workflow-docs.js && node --check ./tests/contracts/workflow-failure-matrix.js && node --check ./tests/contracts/workflow-failure-result.js && node --check ./tests/contracts/workflow-migration.js && node --check ./tests/contracts/workflow-progress.js && node --check ./tests/contracts/workflow-release.js && node --check ./tests/contracts/workflow-review.js && node --check ./tests/contracts/workflow-lifecycle.js && node --check ./tests/contracts/workflow-revision.js && node --check ./tests/contracts/workflow-templates.js && node --check ./tests/contracts/workflow-worker-matrix.js", "check:mcp": "node --check ./bin/cewp-mcp.js && node --check ./src/mcp/server.js && node --check ./src/mcp/tools.js && node --check ./tests/contracts/integration-mcp.js", - "check:evidence": "node --check ./src/evidence/receipt.js && node --check ./src/evidence/events.js && node --check ./src/evidence/report.js && node --check ./src/evidence/compare.js && node --check ./src/evidence/redaction.js && node --check ./src/evidence/verify.js && node --check ./src/run/verify.js && node --check ./tests/contracts/evidence-receipt.js && node --check ./tests/contracts/event-run-verify.js && node --check ./tests/contracts/operator-report.js && node --check ./tests/contracts/run-comparison.js && node --check ./tests/contracts/evidence-redaction.js", + "check:evidence": "node --check ./src/evidence/receipt.js && node --check ./src/evidence/events.js && node --check ./src/evidence/report.js && node --check ./src/evidence/compare.js && node --check ./src/evidence/redaction.js && node --check ./src/evidence/usage.js && node --check ./src/evidence/verify.js && node --check ./src/run/verify.js && node --check ./tests/contracts/evidence-receipt.js && node --check ./tests/contracts/event-run-verify.js && node --check ./tests/contracts/operator-report.js && node --check ./tests/contracts/run-comparison.js && node --check ./tests/contracts/evidence-redaction.js", "check": "npm run check:syntax && npm run check:workflow-syntax && npm run check:mcp && npm run check:evidence && npm test", "pack:dry-run": "npm pack --dry-run" }, diff --git a/src/evidence/receipt.js b/src/evidence/receipt.js index 7f54a0d..30ac639 100644 --- a/src/evidence/receipt.js +++ b/src/evidence/receipt.js @@ -7,6 +7,7 @@ const { getGitHeadCommit } = require("../lib/git"); const { normalizeSlashPath } = require("../lib/paths"); const { loadHostBinding, loadIntegrationControlReceipt } = require("../integration/binding"); const { parseLifecycleEvents } = require("./events"); +const { buildUsageObservations, unknownUsageEstimate } = require("./usage"); const { writeJsonAtomic } = require("../workflow/state"); const EVIDENCE_RECEIPT_SCHEMA_VERSION = "evidence-receipt/v1"; @@ -220,12 +221,18 @@ function buildEvidenceReceipt(found, options = {}) { const resultsByTask = new Map(results.map((entry) => [entry.value.taskId, entry.value])); const reviewValues = reviews.map((entry) => entry.value); const usageValues = [...results.map((entry) => entry.value), ...reviewValues].map((entry) => entry.usage); + const usageObservations = buildUsageObservations( + found, + results.map((entry) => entry.value), + reviewValues, + warnings, + ); let complete = found.run.status === "finalized" && found.run.tasks.every((task) => task.status === "completed" && task.verification && task.verification.status === "passed") && (!found.run.reviewerPolicy.requiredForFinalize || found.run.reviewer.status === "passed"); if (!complete) warnings.push({ code: "run-not-finalized", message: `Run status ${found.run.status} produces a partial receipt.` }); const integrityFiles = integrityInventory(found, results, reviews, warnings); - if (warnings.some((warning) => ["missing-referenced-evidence", "malformed-evidence-file", "malformed-event", "incompatible-event-schema", "invalid-event", "missing-events"].includes(warning.code))) { + if (warnings.some((warning) => ["missing-referenced-evidence", "malformed-evidence-file", "malformed-event", "incompatible-event-schema", "invalid-event", "missing-events", "malformed-usage-observation-ledger"].includes(warning.code))) { complete = false; } const headCommit = getGitHeadCommit(found.repoRoot); @@ -267,8 +274,9 @@ function buildEvidenceReceipt(found, options = {}) { capturedOutputBytes: truthAggregate(usageValues.map((usage) => usage && usage.capturedOutputBytes), "no uniformly observed captured output evidence"), managedTokens: truthAggregate(usageValues.map((usage) => usage && usage.managedTokens), "managed token totals are unavailable"), hostInternal: truthAggregate(usageValues.map((usage) => usage && usage.hostInternal), "host-internal usage is unavailable"), + observations: usageObservations, }, - estimate: { schemaVersion: "usage-estimate/v1", label: "unknown", range: null, confidence: "unavailable", sampleBasis: null, driftState: "unknown" }, + estimate: unknownUsageEstimate(), cost: { apiEquivalent: { label: "unknown", value: null, currency: null, pricingDate: null, model: null, reason: "no valid dated API pricing mapping" } }, warningSurface: { status: "unknown", deliveries: [] }, git: { diff --git a/src/evidence/usage.js b/src/evidence/usage.js new file mode 100644 index 0000000..050aa52 --- /dev/null +++ b/src/evidence/usage.js @@ -0,0 +1,139 @@ +"use strict"; + +const { readHostObservations } = require("../integration/observation"); + +const USAGE_OBSERVATION_SCHEMA_VERSION = "usage-observation/v1"; +const USAGE_ESTIMATE_SCHEMA_VERSION = "usage-estimate/v1"; +const RESULT_CATEGORIES = Object.freeze([ + "managedOperations", + "capturedOutputBytes", + "managedTokens", + "hostInternal", +]); + +function sourceBoundary(source) { + if (typeof source !== "string") return "unknown"; + if (source.startsWith("codex-exec")) return "managed-child-process"; + if (source.startsWith("cewp-")) return "cewp-core-local"; + return "unknown"; +} + +function resultObservations(kind, values) { + return values.flatMap((value) => RESULT_CATEGORIES.map((category) => { + const truth = value.usage[category]; + const sourceId = truth.source || `${kind}-contract`; + return { + schemaVersion: USAGE_OBSERVATION_SCHEMA_VERSION, + observationId: `${kind}:${kind === "task-result" ? value.resultId : value.reviewId}:${category}`, + observedAt: value.completedAt, + scope: { + runId: value.runId, + taskId: value.taskId || (value.scope && value.scope.taskId) || null, + checkpointId: value.checkpointId || (value.scope && value.scope.checkpointId) || null, + }, + category, + rawCategory: `usage.${category}`, + availability: truth.label === "observed" ? "observed" : "unknown", + evidenceClass: truth.label === "observed" ? "observed" : "unknown", + value: truth.value, + reason: truth.reason || null, + source: { + kind, + id: sourceId, + schemaVersion: value.schemaVersion, + authenticationBoundary: sourceBoundary(truth.source), + }, + effectiveModel: { status: "unknown", value: null, reason: "result contract does not expose effective model" }, + rawValue: truth, + }; + })); +} + +function hostObservations(found, warnings) { + let entries; + try { + entries = readHostObservations(found); + } catch (error) { + warnings.push({ code: "malformed-usage-observation-ledger", message: error.message }); + return []; + } + return entries.map((entry) => ({ + schemaVersion: USAGE_OBSERVATION_SCHEMA_VERSION, + observationId: `host:${entry.observationId}`, + observedAt: entry.observedAt, + scope: { + runId: entry.scope.runId, + taskId: entry.scope.taskId, + checkpointId: entry.scope.checkpointId, + }, + category: entry.category, + rawCategory: entry.rawCategory, + availability: entry.availability, + evidenceClass: entry.evidenceClass, + value: entry.data, + reason: entry.reason, + source: { + kind: "host-observation", + id: entry.source.path, + schemaVersion: entry.source.schemaVersion, + codexVersion: entry.source.codexVersion, + authenticationBoundary: entry.source.authenticationBoundary, + }, + effectiveModel: { status: "unknown", value: null, reason: "host observation does not expose effective model" }, + rawValue: null, + billingImpact: entry.billingImpact, + })); +} + +function buildUsageObservations(found, results, reviews, warnings = []) { + return [ + ...resultObservations("task-result", results), + ...resultObservations("review-result", reviews), + ...hostObservations(found, warnings), + ].sort((left, right) => left.observationId < right.observationId ? -1 : left.observationId > right.observationId ? 1 : 0); +} + +function unknownUsageEstimate(reason = "At least five comparable local runs with known model and effort are required.") { + return { + schemaVersion: USAGE_ESTIMATE_SCHEMA_VERSION, + label: "unknown", + range: null, + confidence: "unavailable", + estimator: { + version: "local-history/v1", + method: "comparable-run-interval", + minimumSampleCount: 5, + groupingDimensions: [ + "taskClass", "effectiveModel", "effectiveEffort", "assurance", "repositorySizeBucket", + "checkpointCount", "workerReviewerShape", "repairs", "verificationSchedule", + ], + }, + sampleBasis: { + count: 0, + comparableRunIds: [], + localOnly: true, + promptsStored: false, + sourceStored: false, + rawLogsStored: false, + }, + calibrationSnapshot: { + id: null, + createdAt: null, + intervalCoverage: null, + absoluteError: null, + }, + drift: { + state: "unknown", + checkedAt: null, + changedDimensions: [], + }, + reason, + }; +} + +module.exports = { + USAGE_ESTIMATE_SCHEMA_VERSION, + USAGE_OBSERVATION_SCHEMA_VERSION, + buildUsageObservations, + unknownUsageEstimate, +}; diff --git a/src/integration/observation.js b/src/integration/observation.js index 99e3f15..40b7fbf 100644 --- a/src/integration/observation.js +++ b/src/integration/observation.js @@ -2,6 +2,7 @@ const fs = require("node:fs"); const path = require("node:path"); +const { appendLifecycleEvent } = require("../evidence/events"); const { validateCodexCapabilitySnapshot } = require("./capabilities"); const HOST_OBSERVATION_SCHEMA_VERSION = "host-observation/v1"; @@ -353,6 +354,16 @@ function recordHostObservation(found, candidate, options = {}) { const filePath = ledgerPath(found); fs.mkdirSync(path.dirname(filePath), { recursive: true }); fs.appendFileSync(filePath, `${JSON.stringify(observation)}\n`); + appendLifecycleEvent(found.runRoot, { + timestamp: observation.observedAt, + type: "usage-observed", + runId: found.run.runId, + observationId: observation.observationId, + usageCategory: observation.category, + availability: observation.availability, + evidenceClass: observation.evidenceClass, + actor: observation.source.path, + }); return observation; } diff --git a/tests/contracts/evidence-receipt.js b/tests/contracts/evidence-receipt.js index f68239b..462d99d 100644 --- a/tests/contracts/evidence-receipt.js +++ b/tests/contracts/evidence-receipt.js @@ -19,6 +19,8 @@ const { renderEvidenceReceiptMarkdown, writeEvidenceReceipt, } = require("../../src/evidence/receipt"); +const { recordHostObservation } = require("../../src/integration/observation"); +const { supportedSnapshot } = require("./integration-capabilities"); const cewpCli = path.join(__dirname, "..", "..", "bin", "cewp.js"); @@ -81,6 +83,40 @@ function runContract() { const repoRoot = makeTempRepo("cewp-evidence-receipt-"); try { const found = completeRun(repoRoot); + recordHostObservation(found, { + schemaVersion: "host-observation/v1", + observationId: "receipt-usage-0001", + observedAt: "2026-07-18T12:06:30.000Z", + source: { + path: "codex-exec", + codexVersion: "codex-cli 0.137.0", + schemaVersion: "codex-exec-jsonl/v1", + authenticationBoundary: "managed-child", + }, + scope: { kind: "workflow-run", runId: found.run.runId, taskId: null, checkpointId: null }, + category: "thread-usage", + rawCategory: "turn.completed.usage", + availability: "observed", + data: { inputTokens: 100, cachedInputTokens: 40, outputTokens: 20, reasoningOutputTokens: 5 }, + raw: { input_tokens: 100, cached_input_tokens: 40, output_tokens: 20, reasoning_output_tokens: 5 }, + }, { capabilities: supportedSnapshot() }); + recordHostObservation(found, { + schemaVersion: "host-observation/v1", + observationId: "receipt-usage-imported-0001", + observedAt: "2026-07-18T12:06:31.000Z", + source: { + path: "audit-import", + codexVersion: null, + schemaVersion: "external-receipt/v1", + authenticationBoundary: "external-owner", + }, + scope: { kind: "workflow-run", runId: found.run.runId, taskId: null, checkpointId: null }, + category: "thread-usage", + rawCategory: "external.usage", + availability: "imported", + data: { inputTokens: 50, cachedInputTokens: 0, outputTokens: 10, reasoningOutputTokens: 0 }, + raw: { imported_total: 60 }, + }, { capabilities: supportedSnapshot() }); writeFile(path.join(found.runRoot, "adapter-output", "prompt.md"), "TOP_SECRET_PROMPT\n"); const options = { generatedAt: "2026-07-18T12:07:00.000Z" }; const first = buildEvidenceReceipt(found, options); @@ -97,6 +133,17 @@ function runContract() { assert(first.reviewer.decision === "PASS", "receipt retains independent reviewer PASS"); assert(first.usage.managedOperations.label === "observed", "receipt aggregates observed managed operations"); assert(first.usage.hostInternal.label === "unknown", "unavailable host usage remains unknown"); + assert(first.usage.observations.every((entry) => entry.schemaVersion === "usage-observation/v1"), "usage observations are independently versioned"); + const hostUsage = first.usage.observations.find((entry) => entry.rawCategory === "turn.completed.usage"); + assert(hostUsage.source.authenticationBoundary === "managed-child", "usage observation preserves authentication boundary"); + assert(hostUsage.value.cachedInputTokens === 40 && hostUsage.value.reasoningOutputTokens === 5, "raw usage categories remain distinct"); + assert(hostUsage.rawValue === null, "receipt excludes raw host payload while preserving normalized categories"); + const importedUsage = first.usage.observations.find((entry) => entry.rawCategory === "external.usage"); + assert(importedUsage.availability === "imported" && importedUsage.evidenceClass === "imported", "imported usage is never relabeled observed"); + assert(importedUsage.billingImpact === "unknown", "host usage never implies a billing impact"); + assert(first.events.some((entry) => entry.type === "usage-observed" && entry.category === "usage-observation"), "usage recording emits the versioned lifecycle category"); + assert(first.estimate.estimator.version === "local-history/v1" && first.estimate.sampleBasis.count === 0, "unknown estimate remains reproducible from its empty sample basis"); + assert(first.estimate.calibrationSnapshot.intervalCoverage === null && first.estimate.drift.state === "unknown", "estimate calibration and drift stay explicit"); assert(first.budget.compliance.status === "passed", "receipt proves the approved budget ceilings were respected"); assert(first.budget.compliance.protectedAllocationsRespected === true, "receipt proves protected allocations were not overspent"); assert(first.cost.apiEquivalent.label === "unknown", "currency cost is not invented"); @@ -123,6 +170,10 @@ function runContract() { const missingEvidence = buildEvidenceReceipt(found, options); assert(missingEvidence.completeness.status === "partial", "missing referenced evidence closes receipt completeness"); assert(missingEvidence.warnings.some((warning) => warning.code === "missing-referenced-evidence"), "missing evidence has an actionable warning"); + fs.appendFileSync(path.join(found.runRoot, "integration", "host-observations.jsonl"), "{malformed\n"); + const malformedUsage = buildEvidenceReceipt(found, options); + assert(malformedUsage.completeness.status === "partial", "malformed usage ledger cannot produce a complete receipt"); + assert(malformedUsage.warnings.some((warning) => warning.code === "malformed-usage-observation-ledger"), "malformed usage ledger has an explicit warning"); const partialRepo = makeTempRepo("cewp-evidence-receipt-partial-"); try { From 4b675ecc438f4f554d522020d5660eeeeec56951 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tuncay=20=C3=96lmez?= Date: Fri, 24 Jul 2026 10:00:00 +0300 Subject: [PATCH 23/40] Expand lifecycle recovery evidence --- docs/evidence-receipts.md | 7 + docs/release-notes.md | 1 + src/evidence/receipt.js | 6 + src/workflow/state.js | 184 ++++++++++++++++++++- tests/contracts/event-run-verify.js | 7 +- tests/contracts/evidence-receipt.js | 3 + tests/contracts/integration-binding.js | 5 + tests/contracts/workflow-budget.js | 14 ++ tests/contracts/workflow-failure-result.js | 12 ++ tests/contracts/workflow-lifecycle.js | 2 + 10 files changed, 236 insertions(+), 5 deletions(-) diff --git a/docs/evidence-receipts.md b/docs/evidence-receipts.md index eaca93c..3dcf756 100644 --- a/docs/evidence-receipts.md +++ b/docs/evidence-receipts.md @@ -52,6 +52,9 @@ New workflow lifecycle records use `event/v1`, with a closed type-to-category vo revision, task, checkpoint, dispatch, intervention, verification, usage, estimates, budgets, warnings, pauses, scope, review, cancellation, and finalization. Existing `workflow-event/v1` records are accepted as read-only legacy input and normalized in receipts; CEWP does not rewrite historical ledgers implicitly. +Core workflow transitions emit distinct budget approval, checkpoint, dispatch, scope, verification, usage, +allocation consumption, threshold/warning, pause, cancellation, review, and finalization records where the +corresponding action occurs. `cewp run verify ` checks canonical state/definition consistency, event syntax and schemas, required result/checkpoint artifacts, bound-worktree liveness, and every receipt integrity hash. It executes @@ -69,6 +72,10 @@ The report separates observed, estimated, budgeted, and unknown values; shows ta checkpoint verification, interventions and recovery state, protected reserves, preventive versus observed controls, and final review. Repository metadata is HTML-escaped, and raw prompts/logs remain excluded. +Task receipts retain the failed checkpoint/classification, blocker, failure history, state history, and the +operator intervention/reason that reopened work. Budget-paused receipts remain partial and separately prove +absolute-ceiling and protected-allocation compliance; a pause is never rendered as completion. + ## Run Comparison `cewp workflow compare ` derives `run-comparison/v1` from two receipts. It diff --git a/docs/release-notes.md b/docs/release-notes.md index d5cdf8f..f528940 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -12,6 +12,7 @@ - Added `run-comparison/v1` and `cewp workflow compare` with explicit unknowns and evidence-backed native-goal baseline labeling. - Added adversarially tested `redaction-policy/v1` exports that preserve canonical local evidence and avoid absolute-path output. - Added `usage-observation/v1` provenance/raw-category records and reproducible unknown `usage-estimate/v1` calibration metadata. +- Expanded lifecycle evidence and recovery receipts for failed checkpoints, budget pauses, host limits, cancellations, and audit-only controls. ## 0.11.0-beta.0 diff --git a/src/evidence/receipt.js b/src/evidence/receipt.js index 30ac639..b25b538 100644 --- a/src/evidence/receipt.js +++ b/src/evidence/receipt.js @@ -165,6 +165,12 @@ function taskReceipt(found, runtimeTask, resultsByTask) { failure: result ? result.failure : null, artifacts: result ? result.artifacts : [], resultId: runtimeTask.resultId, + recovery: { + blocker: runtimeTask.blocker, + failureHistory: runtimeTask.failureHistory || [], + stateHistory: runtimeTask.stateHistory || [], + interventions: (found.run.interventions || []).filter((entry) => entry.taskId === runtimeTask.id), + }, }; } diff --git a/src/workflow/state.js b/src/workflow/state.js index 685c0a3..ebca6f9 100644 --- a/src/workflow/state.js +++ b/src/workflow/state.js @@ -349,6 +349,23 @@ function pauseForWorkflowBudget(found, allocation, decision, timestamp) { }; writeJsonAtomic(found.runPath, run); writeWorkflowProgress(found.runRoot, run, found.definition, { now: new Date(timestamp) }); + if (decision.warning) { + appendWorkflowEvent(found.runRoot, { + timestamp, + type: "budget-threshold", + runId: run.runId, + threshold: decision.warning, + percent: decision.percent, + allocation, + }); + appendWorkflowEvent(found.runRoot, { + timestamp, + type: "warning-presented", + runId: run.runId, + warning: decision.warning, + surface: "cewp-core-state", + }); + } appendWorkflowEvent(found.runRoot, { schemaVersion: "workflow-event/v1", timestamp, @@ -417,10 +434,9 @@ function startWorkflowTask(found, taskId, options = {}) { throw new Error(`Controlled workflow operation paused: ${budgetDecision.pauseStatus} (${budgetDecision.reason}).`); } let runBeforeStart = found.run; - if ( - budgetDecision.warning - && !found.run.budget.thresholdEvents.some((event) => event.threshold === budgetDecision.warning) - ) { + const newBudgetWarning = budgetDecision.warning + && !found.run.budget.thresholdEvents.some((event) => event.threshold === budgetDecision.warning); + if (newBudgetWarning) { runBeforeStart = { ...found.run, budget: { @@ -497,6 +513,41 @@ function startWorkflowTask(found, taskId, options = {}) { }; writeJsonAtomic(found.runPath, run); const progress = writeWorkflowProgress(found.runRoot, run, found.definition, { now }); + if (newBudgetWarning) { + appendWorkflowEvent(found.runRoot, { + timestamp, + type: "budget-threshold", + runId: run.runId, + threshold: budgetDecision.warning, + percent: budgetDecision.percent, + allocation, + }); + appendWorkflowEvent(found.runRoot, { + timestamp, + type: "warning-presented", + runId: run.runId, + warning: budgetDecision.warning, + surface: "cewp-core-state", + }); + } + appendWorkflowEvent(found.runRoot, { + timestamp, + type: "checkpoint-started", + runId: run.runId, + taskId, + checkpointId, + attempt, + }); + appendWorkflowEvent(found.runRoot, { + timestamp, + type: "dispatch-started", + runId: run.runId, + taskId, + checkpointId, + workerId, + executionOwner: run.execution.owner, + backend: run.execution.backend, + }); appendWorkflowEvent(found.runRoot, { schemaVersion: "workflow-event/v1", timestamp, @@ -586,6 +637,50 @@ function recordWorkflowFailureResult(found, runtimeTask, checkpoint, checkpointP writeJsonAtomic(found.runPath, run); const now = new Date(result.completedAt); const progress = writeWorkflowProgress(found.runRoot, run, found.definition, { now }); + appendWorkflowEvent(found.runRoot, { + timestamp: result.completedAt, + type: "scope-evaluated", + runId: run.runId, + taskId: runtimeTask.id, + checkpointId: checkpoint.checkpointId, + resultId: result.resultId, + status: "passed", + }); + appendWorkflowEvent(found.runRoot, { + timestamp: result.completedAt, + type: "verification-completed", + runId: run.runId, + taskId: runtimeTask.id, + checkpointId: checkpoint.checkpointId, + resultId: result.resultId, + status: "failed", + }); + appendWorkflowEvent(found.runRoot, { + timestamp: result.completedAt, + type: "usage-observed", + runId: run.runId, + taskId: runtimeTask.id, + checkpointId: checkpoint.checkpointId, + resultId: result.resultId, + usageCategories: Object.fromEntries(Object.entries(result.usage).map(([name, value]) => [name, value.label])), + }); + appendWorkflowEvent(found.runRoot, { + timestamp: result.completedAt, + type: "allocation-consumed", + runId: run.runId, + taskId: runtimeTask.id, + checkpointId: checkpoint.checkpointId, + allocation: checkpoint.budget.activeAllocation, + consumed: budget.consumed.allocations[checkpoint.budget.activeAllocation], + }); + appendWorkflowEvent(found.runRoot, { + timestamp: result.completedAt, + type: "checkpoint-completed", + runId: run.runId, + taskId: runtimeTask.id, + checkpointId: checkpoint.checkpointId, + status: blockedCheckpoint.status, + }); appendWorkflowEvent(found.runRoot, { schemaVersion: "workflow-event/v1", timestamp: result.completedAt, @@ -796,6 +891,50 @@ function recordWorkflowResult(found, taskId, candidate) { }; writeJsonAtomic(found.runPath, run); const progress = writeWorkflowProgress(found.runRoot, run, found.definition, { now: new Date(result.completedAt) }); + appendWorkflowEvent(found.runRoot, { + timestamp: result.completedAt, + type: "scope-evaluated", + runId: run.runId, + taskId, + checkpointId: checkpoint.checkpointId, + resultId: result.resultId, + status: "passed", + }); + appendWorkflowEvent(found.runRoot, { + timestamp: result.completedAt, + type: "verification-completed", + runId: run.runId, + taskId, + checkpointId: checkpoint.checkpointId, + resultId: result.resultId, + status: "passed", + }); + appendWorkflowEvent(found.runRoot, { + timestamp: result.completedAt, + type: "usage-observed", + runId: run.runId, + taskId, + checkpointId: checkpoint.checkpointId, + resultId: result.resultId, + usageCategories: Object.fromEntries(Object.entries(result.usage).map(([name, value]) => [name, value.label])), + }); + appendWorkflowEvent(found.runRoot, { + timestamp: result.completedAt, + type: "allocation-consumed", + runId: run.runId, + taskId, + checkpointId: checkpoint.checkpointId, + allocation: checkpoint.budget.activeAllocation, + consumed: budget.consumed.allocations[checkpoint.budget.activeAllocation], + }); + appendWorkflowEvent(found.runRoot, { + timestamp: result.completedAt, + type: "checkpoint-completed", + runId: run.runId, + taskId, + checkpointId: checkpoint.checkpointId, + status: completedCheckpoint.status, + }); appendWorkflowEvent(found.runRoot, { schemaVersion: "workflow-event/v1", timestamp: result.completedAt, @@ -1268,6 +1407,25 @@ function interveneWorkflowRun(found, options, timestamp, reason) { runId: run.runId, ...intervention, }); + if (options.event === "add-budget") { + appendWorkflowEvent(found.runRoot, { + timestamp, + type: "budget-approved", + runId: run.runId, + actor: intervention.actor, + allocation: options.allocation, + operations: options.operations, + revision: budget.revisions.length, + }); + } else if (options.event === "pause-host-limit") { + appendWorkflowEvent(found.runRoot, { + timestamp, + type: "paused-host-limit", + runId: run.runId, + reason, + actor: intervention.actor, + }); + } return { run, checkpoint: null, @@ -1347,6 +1505,15 @@ function interveneWorkflowLifecycle(found, options, timestamp, reason) { runId: run.runId, ...intervention, }); + if (options.event === "cancel") { + appendWorkflowEvent(found.runRoot, { + timestamp, + type: "workflow-cancelled", + runId: run.runId, + reason, + actor: intervention.actor, + }); + } return { run, checkpoints, @@ -1573,6 +1740,15 @@ function createApprovedRun(options) { approvalDigest, actor: "operator", }); + appendWorkflowEvent(runRoot, { + timestamp, + type: "budget-approved", + runId, + actor: "operator", + budgetSchemaVersion: run.budget.schemaVersion, + modelOperations: run.budget.modelOperations, + protectedAllocations: run.budget.protectedAllocations, + }); writeWorkflowProgress(runRoot, run, options.definition, { now }); return { run, diff --git a/tests/contracts/event-run-verify.js b/tests/contracts/event-run-verify.js index 4ad9456..6cdbeb9 100644 --- a/tests/contracts/event-run-verify.js +++ b/tests/contracts/event-run-verify.js @@ -37,9 +37,10 @@ function runContract() { let found = loadWorkflowRun(repoRoot, approved.runId); const eventsPath = path.join(found.runRoot, "events.jsonl"); const events = readLifecycleEvents(eventsPath, { runId: approved.runId }); - assert(events.length === 1, "approved workflow writes one initial lifecycle event"); + assert(events.length === 2, "approved workflow writes run and budget approval events"); assert(events[0].schemaVersion === EVENT_SCHEMA_VERSION, "new lifecycle events use event/v1"); assert(events[0].category === "run", "workflow approval is categorized as a run event"); + assert(events[1].category === "budget-approval", "approved envelope is a distinct budget event"); const legacy = normalizeLifecycleEvent({ schemaVersion: "workflow-event/v1", @@ -53,6 +54,10 @@ function runContract() { const startedAt = new Date(new Date(found.run.createdAt).getTime() + 1000); startWorkflowTask(found, "implement-example", { now: startedAt }); found = loadWorkflowRun(repoRoot, approved.runId); + const startedEvents = readLifecycleEvents(eventsPath, { runId: approved.runId }); + for (const category of ["checkpoint", "dispatch", "task"]) { + assert(startedEvents.some((entry) => entry.category === category), `task start emits ${category} lifecycle evidence`); + } writeEvidenceReceipt(found, { generatedAt: "2026-07-22T10:01:00.000Z" }); const healthy = verifyWorkflowRun(repoRoot, approved.runId); assert(healthy.schemaVersion === "run-verification/v1", "run verification is versioned"); diff --git a/tests/contracts/evidence-receipt.js b/tests/contracts/evidence-receipt.js index 462d99d..5bde765 100644 --- a/tests/contracts/evidence-receipt.js +++ b/tests/contracts/evidence-receipt.js @@ -130,6 +130,9 @@ function runContract() { assert(first.tasks.length === 1 && first.checkpoints.length === 1, "receipt explains task and checkpoint structure"); assert(first.tasks[0].changedFiles.includes("src/example.js"), "receipt records changed files"); assert(first.tasks[0].verification.targeted[0].status === "passed", "receipt records verification evidence"); + for (const category of ["scope", "verification", "usage-observation", "allocation-consumption", "checkpoint"]) { + assert(first.events.some((entry) => entry.category === category), `completed task emits ${category} lifecycle evidence`); + } assert(first.reviewer.decision === "PASS", "receipt retains independent reviewer PASS"); assert(first.usage.managedOperations.label === "observed", "receipt aggregates observed managed operations"); assert(first.usage.hostInternal.label === "unknown", "unavailable host usage remains unknown"); diff --git a/tests/contracts/integration-binding.js b/tests/contracts/integration-binding.js index 5b4d922..0e55c08 100644 --- a/tests/contracts/integration-binding.js +++ b/tests/contracts/integration-binding.js @@ -14,6 +14,7 @@ const { loadHostBinding, } = require("../../src/integration/binding"); const { loadWorkflowRun, startWorkflowTask } = require("../../src/workflow/state"); +const { buildEvidenceReceipt } = require("../../src/evidence/receipt"); const cewpCli = path.join(__dirname, "..", "..", "bin", "cewp.js"); @@ -186,6 +187,10 @@ function main() { controlReceipt.claims.providerExecutionSuppliesEnforcement === false, "audit receipt never treats provider-controlled execution as the enforcement source", ); + const auditEvidence = buildEvidenceReceipt(auditFound, { generatedAt: "2026-07-22T15:20:00.000Z" }); + const importedControl = auditEvidence.policy.controls.controls.find((entry) => entry.classification === "imported"); + assert(importedControl.effect === "observed-not-enforced", "audit-only evidence receipt preserves observed-not-enforced effect"); + assert(auditEvidence.policy.controls.summary.preventiveEnforced === 0, "audit-only evidence receipt never reports preventive enforcement"); const shown = runNode(cewpCli, ["integration", "controls", audit.runId, "--json"], auditRepo); assert(shown.status === 0, `control receipt is available through operator JSON: ${shown.stderr}`); const shownReceipt = JSON.parse(shown.stdout); diff --git a/tests/contracts/workflow-budget.js b/tests/contracts/workflow-budget.js index 165b4c5..27e0063 100644 --- a/tests/contracts/workflow-budget.js +++ b/tests/contracts/workflow-budget.js @@ -1,11 +1,14 @@ "use strict"; +const fs = require("node:fs"); const path = require("node:path"); const { assert } = require("../harness/lib/assertions"); const { evaluateWorkflowOperation } = require("../../src/workflow/budget"); const { validDefinition } = require("./workflow-definition"); const { successfulResult } = require("./workflow-result"); const { approveWorkflow } = require("./workflow-scheduler"); +const { loadWorkflowRun } = require("../../src/workflow/state"); +const { buildEvidenceReceipt } = require("../../src/evidence/receipt"); const { cleanupRepo, makeTempRepo, @@ -119,6 +122,15 @@ function runWorkflowBudgetContract() { ], repoRoot).stdout).data.run; assert(paused.status === "paused-budget-safe", "budget refusal persists a safe pause"); assert(paused.budget.pauseReason === "implementation-allocation-exhausted", "pause reason is canonical state"); + const pausedReceipt = buildEvidenceReceipt(loadWorkflowRun(repoRoot, approved.runId), { + generatedAt: "2026-07-22T15:10:00.000Z", + }); + assert(pausedReceipt.completeness.status === "partial" && pausedReceipt.budget.pauseReason === "implementation-allocation-exhausted", "budget-paused run has an explanatory partial receipt"); + assert(pausedReceipt.budget.compliance.absoluteCeilingRespected === true, "paused receipt proves the absolute ceiling was respected"); + assert(pausedReceipt.budget.compliance.protectedAllocationsRespected === true, "paused receipt proves protected allocations were preserved"); + assert(pausedReceipt.events.some((entry) => entry.category === "safe-pause"), "safe pause is normalized in the event ledger"); + assert(pausedReceipt.events.some((entry) => entry.category === "threshold"), "budget refusal records the reached threshold"); + assert(pausedReceipt.events.some((entry) => entry.category === "warning-presentation"), "budget refusal records Core warning presentation"); const addBudget = runNode(cewpCli, [ "workflow", "intervene", approved.runId, @@ -152,6 +164,8 @@ function runWorkflowBudgetContract() { const hostPaused = JSON.parse(hostPause.stdout).data.run; assert(hostPaused.status === "paused-host-limit", "host limit has a distinct run state"); assert(hostPaused.budget.hostLimit.active === true, "host limit observation is retained"); + const hostPauseEvents = fs.readFileSync(path.join(repoRoot, ".cewp", "workflow-runs", approved.runId, "events.jsonl"), "utf8"); + assert(hostPauseEvents.includes("\"category\":\"host-limit\""), "manual host pause emits the host-limit lifecycle category"); const hostResume = runNode(cewpCli, [ "workflow", "intervene", approved.runId, "--event", "resume", diff --git a/tests/contracts/workflow-failure-result.js b/tests/contracts/workflow-failure-result.js index 1888ed0..e70fa78 100644 --- a/tests/contracts/workflow-failure-result.js +++ b/tests/contracts/workflow-failure-result.js @@ -11,6 +11,8 @@ const { } = require("../harness/lib/temp-repo"); const { validDefinition } = require("./workflow-definition"); const { approveWorkflow } = require("./workflow-scheduler"); +const { loadWorkflowRun } = require("../../src/workflow/state"); +const { buildEvidenceReceipt } = require("../../src/evidence/receipt"); const cewpCli = path.join(__dirname, "..", "..", "bin", "cewp.js"); @@ -109,6 +111,16 @@ function runWorkflowFailureResultContract() { const firstRetry = retry(repoRoot, run.runId); assert(firstRetry.status === 0, `first failure permits bounded retry: ${firstRetry.stderr}`); + const recoveryReceipt = buildEvidenceReceipt(loadWorkflowRun(repoRoot, run.runId), { + generatedAt: "2026-07-22T15:00:00.000Z", + }); + const recoveredTask = recoveryReceipt.tasks.find((task) => task.id === "implement-example"); + const failedCheckpoint = recoveryReceipt.checkpoints.find((entry) => entry.checkpointId === firstCheckpoint.checkpointId); + assert(recoveryReceipt.completeness.status === "partial", "recovery in progress produces a partial receipt"); + assert(failedCheckpoint.status === "blocked" && failedCheckpoint.failureClassification === "new-regression", "receipt identifies the failed checkpoint and classification"); + assert(recoveredTask.recovery.failureHistory[0].checkpointId === firstCheckpoint.checkpointId, "receipt retains the failure that triggered recovery"); + assert(recoveredTask.recovery.interventions.some((entry) => entry.event === "retry" && entry.reason === "Apply one bounded repair"), "receipt explains why the run continued"); + assert(recoveredTask.status === "ready" && recoveredTask.attempts === 1, "receipt shows retry changed the task back to ready without claiming success"); const secondCheckpoint = startTask(repoRoot, run.runId); assert(secondCheckpoint.budget.activeAllocation === "repair", "retry consumes only repair allocation"); const repeatedResult = recordFailure(repoRoot, run, secondCheckpoint, "implement-example-failure-2"); diff --git a/tests/contracts/workflow-lifecycle.js b/tests/contracts/workflow-lifecycle.js index 309a2f4..988f86b 100644 --- a/tests/contracts/workflow-lifecycle.js +++ b/tests/contracts/workflow-lifecycle.js @@ -56,6 +56,8 @@ function runWorkflowLifecycleContract() { const cancelled = JSON.parse(cancelledResult.stdout).data.run; assert(cancelled.status === "cancelled", "cancel is a terminal non-success run state"); assert(cancelled.tasks.every((task) => task.status === "cancelled"), "cancel cascades to all unfinished tasks"); + const cancellationEvents = fs.readFileSync(path.join(repoRoot, ".cewp", "workflow-runs", cancelledRun.runId, "events.jsonl"), "utf8"); + assert(cancellationEvents.includes("\"category\":\"cancellation\""), "cancellation has a dedicated lifecycle category"); const abandonedResult = intervene(repoRoot, cancelledRun.runId, "abandon", { reason: "Operator closes the cancelled run", }); From ae7cd6265cf7f106b2c0545ca6945443c08bb273 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tuncay=20=C3=96lmez?= Date: Fri, 24 Jul 2026 10:30:00 +0300 Subject: [PATCH 24/40] Expand human evidence receipts --- docs/evidence-receipts.md | 5 ++ docs/release-notes.md | 1 + src/evidence/receipt.js | 108 ++++++++++++++++++++++++++-- tests/contracts/evidence-receipt.js | 7 ++ 4 files changed, 115 insertions(+), 6 deletions(-) diff --git a/docs/evidence-receipts.md b/docs/evidence-receipts.md index 3dcf756..2e36e9e 100644 --- a/docs/evidence-receipts.md +++ b/docs/evidence-receipts.md @@ -23,6 +23,11 @@ documented variable. Historical runs that predate a field retain an explicit unk finalized, have malformed event/evidence data, or are missing referenced evidence produce a partial receipt with warnings rather than a complete claim. +The Markdown view is generated from that same model and includes tasks, checkpoints, approved commands and +verification, revisions, interventions/recovery, budget and protected reserves, usage provenance/estimate, +control classifications, final review, timestamps, and warnings. It is intended to explain a run without +requiring raw logs; the JSON remains the machine-readable contract. + ## Integrity Boundary Integrity entries contain byte length and `sha256` for canonical run evidence, the approved definition, diff --git a/docs/release-notes.md b/docs/release-notes.md index f528940..34503fc 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -13,6 +13,7 @@ - Added adversarially tested `redaction-policy/v1` exports that preserve canonical local evidence and avoid absolute-path output. - Added `usage-observation/v1` provenance/raw-category records and reproducible unknown `usage-estimate/v1` calibration metadata. - Expanded lifecycle evidence and recovery receipts for failed checkpoints, budget pauses, host limits, cancellations, and audit-only controls. +- Expanded the Markdown receipt so complete and partial runs can be understood without opening raw logs. ## 0.11.0-beta.0 diff --git a/src/evidence/receipt.js b/src/evidence/receipt.js index b25b538..512fc8a 100644 --- a/src/evidence/receipt.js +++ b/src/evidence/receipt.js @@ -329,9 +329,48 @@ function buildEvidenceReceipt(found, options = {}) { } function renderEvidenceReceiptMarkdown(receipt) { - const taskLines = receipt.tasks.map((task) => ( - `- ${task.id}: ${task.status}; attempts ${task.attempts}; scope ${task.scopeVerdict.status}; changed ${task.changedFiles.join(", ") || "none"}` + const taskLines = receipt.tasks.flatMap((task) => { + const recovery = task.recovery || { failureHistory: [], interventions: [] }; + return [ + `- ${task.id}: ${task.status}; attempts ${task.attempts}; scope ${task.scopeVerdict.status}`, + ` - Allowed: ${task.allowedFiles.join(", ") || "none"}`, + ` - Changed: ${task.changedFiles.join(", ") || "none"}`, + ` - Artifacts: ${task.artifacts.map((entry) => `${entry.kind}:${entry.path}`).join(", ") || "none"}`, + ` - Failure: ${task.failure ? `${task.failure.classification}: ${task.failure.summary}` : "none"}`, + ` - Recovery: ${recovery.interventions.map((entry) => `${entry.event}: ${entry.reason}`).join("; ") || "none"}`, + ]; + }).join("\n"); + const checkpointLines = receipt.checkpoints.map((entry) => { + const baseline = entry.verification && entry.verification.baseline + ? entry.verification.baseline.status + : "unknown"; + return `- ${entry.checkpointId}: ${entry.status}; task ${entry.taskId}; attempt ${entry.attempt}; baseline ${baseline}; failure ${entry.failureClassification || "none"}`; + }).join("\n"); + const commandLines = receipt.commands.flatMap((entry) => [ + `- ${entry.taskId} baseline: ${entry.verification.baseline || "none"}`, + ...entry.verification.targeted.map((command) => ` - targeted: ${command}`), + ...entry.verification.full.map((command) => ` - full: ${command}`), + ]).join("\n"); + const revisionLines = receipt.planRevisions.map((entry) => ( + `- revision ${entry.revision}: ${entry.reason || entry.digest}; superseded ${entry.supersededAt || "unknown"}` )).join("\n"); + const interventionLines = receipt.interventions.map((entry) => ( + `- ${entry.event}: ${entry.reason}; actor ${entry.actor}; ${entry.recordedAt}` + )).join("\n"); + const significantEventLines = receipt.events + .filter((entry) => ["threshold", "warning-presentation", "safe-pause", "unverified-pause", "host-limit", "cancellation"].includes(entry.category)) + .map((entry) => `- ${entry.timestamp}: ${entry.category}/${entry.type}; ${entry.reason || entry.warning || entry.threshold || "recorded"}`) + .join("\n"); + const allocationLines = receipt.budget.compliance.allocations.map((entry) => ( + `- ${entry.name}: ${entry.consumed}/${entry.approved}; protected ${entry.protected ? "yes" : "no"}; ${entry.respected ? "passed" : "failed"}` + )).join("\n"); + const observationLines = receipt.usage.observations.map((entry) => ( + `- ${entry.category}/${entry.rawCategory}: ${entry.availability}; source ${entry.source.id}; auth ${entry.source.authenticationBoundary}; model ${entry.effectiveModel.status}` + )).join("\n"); + const controlReceipt = receipt.policy.controls; + const controlLines = controlReceipt + ? controlReceipt.controls.map((entry) => `- ${entry.name}: ${entry.classification}; ${entry.effect}`).join("\n") + : "- none"; const warningLines = receipt.warnings.length > 0 ? receipt.warnings.map((warning) => `- ${warning.code}: ${warning.message}`).join("\n") : "- none"; @@ -341,13 +380,11 @@ function renderEvidenceReceiptMarkdown(receipt) { - Goal: ${receipt.goal} - Status: ${receipt.completeness.runStatus} (${receipt.completeness.status}) - Execution: ${receipt.execution.owner} / ${receipt.execution.backend || "none"} -- Reviewer: ${receipt.reviewer.decision || "none"} +- Operating modes: ${receipt.operatingModes.join(", ")} - Source: ${receipt.sourcePlan.kind}; ${receipt.sourcePlan.path || "direct goal"} +- Workflow: ${receipt.workflow.id} revision ${receipt.workflow.revision}; ${receipt.workflow.digest} - Git base: ${receipt.git.baseCommit.status === "known" ? receipt.git.baseCommit.value : "unknown"} - Git head: ${receipt.git.headCommit.value} -- Managed operations: ${receipt.usage.managedOperations.label}${receipt.usage.managedOperations.value === null ? "" : ` (${receipt.usage.managedOperations.value})`} -- Host-internal usage: ${receipt.usage.hostInternal.label} -- API-equivalent cost: ${receipt.cost.apiEquivalent.label} - Integrity: ${receipt.integrity.claim}; ${receipt.integrity.files.length} hashed files; not tamper-proof ${receipt.redaction && receipt.redaction.applied ? `- Redaction: applied (${receipt.redaction.replacements} replacements)\n` : ""} @@ -355,6 +392,65 @@ ${receipt.redaction && receipt.redaction.applied ? `- Redaction: applied (${rece ${taskLines || "- none"} +## Checkpoints + +${checkpointLines || "- none"} + +## Commands and verification + +${commandLines || "- none"} + +## Plan revisions + +${revisionLines || "- none"} + +## Interventions and lifecycle decisions + +${interventionLines || "- none"} +${significantEventLines ? `\n${significantEventLines}` : ""} + +## Budget + +- Compliance: ${receipt.budget.compliance.status} +- Absolute ceiling: ${receipt.budget.compliance.absoluteCeilingRespected ? "passed" : "failed"} +- Protected allocations: ${receipt.budget.compliance.protectedAllocationsRespected ? "passed" : "failed"} +- Pause reason: ${receipt.budget.pauseReason || "none"} + +${allocationLines || "- none"} + +## Usage and estimate + +- Managed operations: ${receipt.usage.managedOperations.label}${receipt.usage.managedOperations.value === null ? "" : ` (${receipt.usage.managedOperations.value})`} +- Captured output: ${receipt.usage.capturedOutputBytes.label}${receipt.usage.capturedOutputBytes.value === null ? "" : ` (${receipt.usage.capturedOutputBytes.value})`} +- Managed tokens: ${receipt.usage.managedTokens.label} +- Host-internal usage: ${receipt.usage.hostInternal.label} +- Estimate: ${receipt.estimate.label}; confidence ${receipt.estimate.confidence}; samples ${receipt.estimate.sampleBasis.count}; estimator ${receipt.estimate.estimator.version}; drift ${receipt.estimate.drift.state} +- API-equivalent cost: ${receipt.cost.apiEquivalent.label}${receipt.cost.apiEquivalent.model ? `; model ${receipt.cost.apiEquivalent.model}; pricing ${receipt.cost.apiEquivalent.pricingDate}` : ""} + +${observationLines || "- no usage observations"} + +## Controls + +- Execution owner: ${receipt.execution.owner} +- Preventive enforced: ${controlReceipt ? controlReceipt.summary.preventiveEnforced : 0} +- Imported observed: ${controlReceipt ? controlReceipt.summary.importedObserved : 0} + +${controlLines} + +## Final review + +- Status: ${receipt.reviewer.status} +- Decision: ${receipt.reviewer.decision || "none"} +- Review ID: ${receipt.reviewer.reviewId || "none"} +- Reviews recorded: ${receipt.reviews.length} + +## Timestamps + +- Created: ${receipt.timestamps.createdAt} +- Updated: ${receipt.timestamps.updatedAt} +- Finalized: ${receipt.timestamps.finalizedAt || "none"} +- Receipt generated: ${receipt.timestamps.generatedAt} + ## Warnings ${warningLines} diff --git a/tests/contracts/evidence-receipt.js b/tests/contracts/evidence-receipt.js index 5bde765..2b719e7 100644 --- a/tests/contracts/evidence-receipt.js +++ b/tests/contracts/evidence-receipt.js @@ -159,6 +159,13 @@ function runContract() { const markdown = renderEvidenceReceiptMarkdown(first); assert(markdown.includes("# CEWP Evidence Receipt"), "Markdown receipt is human-readable"); assert(markdown.includes("Host-internal usage: unknown"), "Markdown keeps explicit unknowns"); + for (const section of ["## Tasks", "## Checkpoints", "## Commands and verification", "## Budget", "## Usage and estimate", "## Controls", "## Final review", "## Timestamps"]) { + assert(markdown.includes(section), `Markdown receipt includes ${section}`); + } + assert(markdown.includes("implement-example-attempt-0001"), "Markdown identifies checkpoint evidence"); + assert(markdown.includes("node --test tests/example.test.js"), "Markdown identifies approved verification commands"); + assert(markdown.includes("Protected allocations: passed"), "Markdown proves protected reserve compliance"); + assert(markdown.includes("Decision: PASS"), "Markdown exposes independent final review"); assert(!markdown.includes("TOP_SECRET_PROMPT"), "Markdown does not include prompt or raw log content"); const written = writeEvidenceReceipt(found, options); assert(fs.existsSync(written.paths.json) && fs.existsSync(written.paths.markdown), "JSON and Markdown receipts are written locally"); From 984dc9a587de5d897afe9e96808ba8f148fd9b88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tuncay=20=C3=96lmez?= Date: Fri, 24 Jul 2026 11:00:00 +0300 Subject: [PATCH 25/40] Report evidenced providers and warnings --- src/evidence/receipt.js | 19 +++++++++++++++++-- tests/contracts/evidence-receipt.js | 1 + tests/contracts/run-comparison.js | 1 + tests/contracts/workflow-budget.js | 1 + 4 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/evidence/receipt.js b/src/evidence/receipt.js index 512fc8a..61e0afb 100644 --- a/src/evidence/receipt.js +++ b/src/evidence/receipt.js @@ -244,6 +244,16 @@ function buildEvidenceReceipt(found, options = {}) { const headCommit = getGitHeadCommit(found.repoRoot); const baseCommit = found.run.git && found.run.git.baseCommit; const hostBinding = loadHostBinding(found); + const warningDeliveries = events + .filter((entry) => entry.category === "warning-presentation") + .map((entry) => ({ + warning: entry.warning || "unknown", + surface: entry.surface || "unknown", + deliveredAt: entry.timestamp, + evidence: "event/v1", + })); + const knownCodexProvider = found.run.execution.backend === "codex-exec" + || Boolean(hostBinding && hostBinding.host.product === "codex"); return { schemaVersion: EVIDENCE_RECEIPT_SCHEMA_VERSION, @@ -265,7 +275,7 @@ function buildEvidenceReceipt(found, options = {}) { interventions: found.run.interventions || [], events, providers: [{ - provider: found.run.execution.backend === "codex-exec" + provider: knownCodexProvider ? { status: "known", value: "codex" } : { status: "unknown", value: null, reason: "selected execution boundary does not identify a provider" }, effectiveModel: { status: "unknown", value: null, reason: "workflow result does not expose an effective model" }, @@ -284,7 +294,9 @@ function buildEvidenceReceipt(found, options = {}) { }, estimate: unknownUsageEstimate(), cost: { apiEquivalent: { label: "unknown", value: null, currency: null, pricingDate: null, model: null, reason: "no valid dated API pricing mapping" } }, - warningSurface: { status: "unknown", deliveries: [] }, + warningSurface: warningDeliveries.length > 0 + ? { status: "observed", deliveries: warningDeliveries } + : { status: "unknown", deliveries: [] }, git: { baseCommit: baseCommit ? { status: "known", value: baseCommit } @@ -381,6 +393,8 @@ function renderEvidenceReceiptMarkdown(receipt) { - Status: ${receipt.completeness.runStatus} (${receipt.completeness.status}) - Execution: ${receipt.execution.owner} / ${receipt.execution.backend || "none"} - Operating modes: ${receipt.operatingModes.join(", ")} +- Provider: ${receipt.providers[0].provider.status === "known" ? receipt.providers[0].provider.value : "unknown"} +- Effective model: ${receipt.providers[0].effectiveModel.status === "known" ? receipt.providers[0].effectiveModel.value : "unknown"} - Source: ${receipt.sourcePlan.kind}; ${receipt.sourcePlan.path || "direct goal"} - Workflow: ${receipt.workflow.id} revision ${receipt.workflow.revision}; ${receipt.workflow.digest} - Git base: ${receipt.git.baseCommit.status === "known" ? receipt.git.baseCommit.value : "unknown"} @@ -426,6 +440,7 @@ ${allocationLines || "- none"} - Host-internal usage: ${receipt.usage.hostInternal.label} - Estimate: ${receipt.estimate.label}; confidence ${receipt.estimate.confidence}; samples ${receipt.estimate.sampleBasis.count}; estimator ${receipt.estimate.estimator.version}; drift ${receipt.estimate.drift.state} - API-equivalent cost: ${receipt.cost.apiEquivalent.label}${receipt.cost.apiEquivalent.model ? `; model ${receipt.cost.apiEquivalent.model}; pricing ${receipt.cost.apiEquivalent.pricingDate}` : ""} +- Warning surface: ${receipt.warningSurface.status}; deliveries ${receipt.warningSurface.deliveries.length} ${observationLines || "- no usage observations"} diff --git a/tests/contracts/evidence-receipt.js b/tests/contracts/evidence-receipt.js index 2b719e7..87294f9 100644 --- a/tests/contracts/evidence-receipt.js +++ b/tests/contracts/evidence-receipt.js @@ -166,6 +166,7 @@ function runContract() { assert(markdown.includes("node --test tests/example.test.js"), "Markdown identifies approved verification commands"); assert(markdown.includes("Protected allocations: passed"), "Markdown proves protected reserve compliance"); assert(markdown.includes("Decision: PASS"), "Markdown exposes independent final review"); + assert(markdown.includes("Provider: codex") && markdown.includes("Effective model: unknown"), "Markdown distinguishes known provider from unknown effective model"); assert(!markdown.includes("TOP_SECRET_PROMPT"), "Markdown does not include prompt or raw log content"); const written = writeEvidenceReceipt(found, options); assert(fs.existsSync(written.paths.json) && fs.existsSync(written.paths.markdown), "JSON and Markdown receipts are written locally"); diff --git a/tests/contracts/run-comparison.js b/tests/contracts/run-comparison.js index 4a2015f..a9069b9 100644 --- a/tests/contracts/run-comparison.js +++ b/tests/contracts/run-comparison.js @@ -42,6 +42,7 @@ function runContract() { const generatedAt = "2026-07-22T13:00:00.000Z"; const left = buildEvidenceReceipt(loadWorkflowRun(repoRoot, managed.runId), { generatedAt }); const right = buildEvidenceReceipt(loadWorkflowRun(repoRoot, native.runId), { generatedAt }); + assert(right.providers[0].provider.status === "known" && right.providers[0].provider.value === "codex", "validated native Codex binding identifies the provider"); const comparison = compareEvidenceReceipts(left, right); assert(comparison.schemaVersion === "run-comparison/v1", "run comparison is versioned"); assert(comparison.runs.left.execution.owner === "managed" && comparison.runs.right.execution.owner === "native", "execution owner and backend are compared"); diff --git a/tests/contracts/workflow-budget.js b/tests/contracts/workflow-budget.js index 27e0063..a44b3fb 100644 --- a/tests/contracts/workflow-budget.js +++ b/tests/contracts/workflow-budget.js @@ -131,6 +131,7 @@ function runWorkflowBudgetContract() { assert(pausedReceipt.events.some((entry) => entry.category === "safe-pause"), "safe pause is normalized in the event ledger"); assert(pausedReceipt.events.some((entry) => entry.category === "threshold"), "budget refusal records the reached threshold"); assert(pausedReceipt.events.some((entry) => entry.category === "warning-presentation"), "budget refusal records Core warning presentation"); + assert(pausedReceipt.warningSurface.status === "observed" && pausedReceipt.warningSurface.deliveries[0].surface === "cewp-core-state", "receipt reports warning delivery only from event evidence"); const addBudget = runNode(cewpCli, [ "workflow", "intervene", approved.runId, From 1d37bf1848bc956fe5a0ba4122709a53f09e6281 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tuncay=20=C3=96lmez?= Date: Fri, 24 Jul 2026 11:30:00 +0300 Subject: [PATCH 26/40] Prepare Phase 12 beta release surface --- docs/release-notes.md | 25 +++++++++++- package.json | 2 +- plugins/cewp/.codex-plugin/plugin.json | 2 +- tests/contracts/integration-hook-evidence.js | 2 +- tests/contracts/workflow-release.js | 40 ++++++++++---------- tests/harness/run-smoke.js | 2 +- 6 files changed, 48 insertions(+), 25 deletions(-) diff --git a/docs/release-notes.md b/docs/release-notes.md index 34503fc..9c04947 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -2,12 +2,24 @@ ## Unreleased -### Phase 12 development +No changes yet. + +## 0.12.0-beta.0 + +### Summary + +Portable, deterministic workflow evidence and offline operator reporting. This version is prepared locally +and is not published, tagged, or released. Exact clean Linux validation remains required before the Phase 12 +technical gate can close; Windows validation and cross-platform-safe artifact contracts do not substitute +for that run. Numeric usage confidence also remains unavailable until real comparable local samples meet +the documented calibration and drift thresholds. + +### Added - Added deterministic `evidence-receipt/v1` JSON/Markdown generation for workflow runs with complete-versus-partial truth, task/checkpoint/review evidence, usage unknowns, budget compliance, git identities, and local SHA-256 integrity metadata. - Added `cewp workflow receipt ` without executing agents or verification commands. Raw prompts, transcripts, adapter output, and raw log contents remain excluded by default. - Added the closed `event/v1` lifecycle vocabulary with read-only normalization of historical `workflow-event/v1` records. -- Added read-only `cewp run verify ` checks for state, schemas, events, required artifacts, worktree liveness, and receipt integrity. +- Added read-only `cewp run verify` checks for workflow state, schemas, events, required artifacts, worktree liveness, and receipt integrity. - Added portable `operator-report/v1` JSON and standalone offline HTML generation from the normalized receipt model. - Added `run-comparison/v1` and `cewp workflow compare` with explicit unknowns and evidence-backed native-goal baseline labeling. - Added adversarially tested `redaction-policy/v1` exports that preserve canonical local evidence and avoid absolute-path output. @@ -15,6 +27,15 @@ - Expanded lifecycle evidence and recovery receipts for failed checkpoints, budget pauses, host limits, cancellations, and audit-only controls. - Expanded the Markdown receipt so complete and partial runs can be understood without opening raw logs. +### Truth boundaries + +- `usage-observation/v1` keeps source schemas, authentication boundaries, raw category names, and observed/imported/unknown states distinct. +- `usage-estimate/v1` keeps its estimator, sample basis, calibration snapshot, and drift state; insufficient evidence produces no numeric range. +- `run-comparison/v1` compares only equivalent observed dimensions. In particular, unavailable native usage remains unknown rather than zero. +- Audit-only controls remain observed-not-enforced and never appear as preventive enforcement. +- API-equivalent currency cost remains unknown without a supported dated model/pricing mapping. +- No publish, tag, or release action was performed. + ## 0.11.0-beta.0 ### Summary diff --git a/package.json b/package.json index d777533..01fd252 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@setrathex/codex-engineering-workflow-pack", - "version": "0.11.0-beta.0", + "version": "0.12.0-beta.0", "description": "Long-running Codex goals without blind runs: local-first supervision, verification, recovery, review, and evidence.", "license": "MIT", "repository": { diff --git a/plugins/cewp/.codex-plugin/plugin.json b/plugins/cewp/.codex-plugin/plugin.json index 2c21b5c..ef97f19 100644 --- a/plugins/cewp/.codex-plugin/plugin.json +++ b/plugins/cewp/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "cewp", - "version": "0.11.0-beta.0", + "version": "0.12.0-beta.0", "description": "Plan, run, recover, and review bounded engineering checkpoints through the local CEWP runtime.", "author": { "name": "SetraTheXX", diff --git a/tests/contracts/integration-hook-evidence.js b/tests/contracts/integration-hook-evidence.js index 3422e86..a0ec961 100644 --- a/tests/contracts/integration-hook-evidence.js +++ b/tests/contracts/integration-hook-evidence.js @@ -44,7 +44,7 @@ function main() { const output = JSON.parse(approved.stdout); assert(output.command === "integration.hooks.approve", "approval identifies the public command"); assert(output.data.trust.schemaVersion === "codex-hook-trust/v1", "hook trust is versioned"); - assert(output.data.trust.cewpVersion === "0.11.0-beta.0", "approval binds the CEWP runtime version"); + assert(output.data.trust.cewpVersion === "0.12.0-beta.0", "approval binds the CEWP runtime version"); assert(output.data.trust.codexVersion === "codex-cli 0.200.0", "approval binds the observed Codex version"); assert(/^sha256:[a-f0-9]{64}$/.test(output.data.trust.bundleDigest), "approval binds the exact hook bundle"); assert(output.data.nextAction.command === "/hooks", "approval still requires the host trust review"); diff --git a/tests/contracts/workflow-release.js b/tests/contracts/workflow-release.js index 5405ce3..fb78d7c 100644 --- a/tests/contracts/workflow-release.js +++ b/tests/contracts/workflow-release.js @@ -13,38 +13,40 @@ const plugin = JSON.parse(fs.readFileSync( const releaseNotes = fs.readFileSync(path.join(repoRoot, "docs", "release-notes.md"), "utf8"); function runWorkflowReleaseContract() { - assert(packageJson.version === "0.11.0-beta.0", "Phase 11 package version is exact"); + assert(packageJson.version === "0.12.0-beta.0", "Phase 12 package version is exact"); assert(plugin.version === packageJson.version, "plugin version follows the package version"); const unreleasedIndex = releaseNotes.indexOf("## Unreleased"); - const releaseIndex = releaseNotes.indexOf("## 0.11.0-beta.0"); - const previousIndex = releaseNotes.indexOf("## 0.10.0-beta.0"); - assert(unreleasedIndex >= 0 && releaseIndex > unreleasedIndex, "fresh Unreleased precedes the Phase 11 release"); - assert(previousIndex > releaseIndex, "Phase 11 release precedes earlier release history"); + const releaseIndex = releaseNotes.indexOf("## 0.12.0-beta.0"); + const previousIndex = releaseNotes.indexOf("## 0.11.0-beta.0"); + assert(unreleasedIndex >= 0 && releaseIndex > unreleasedIndex, "fresh Unreleased precedes the Phase 12 release"); + assert(previousIndex > releaseIndex, "Phase 12 release precedes earlier release history"); const unreleased = releaseNotes.slice(unreleasedIndex, releaseIndex); - assert(unreleased.includes("Phase 12 development"), "post-Phase 11 work remains in Unreleased"); + assert(unreleased.includes("No changes yet."), "fresh Unreleased is explicitly empty"); for (const claim of [ - "native and managed ownership", - "no automatic model routing", - "SubagentStart", - "eight Core-backed MCP tools", - "observed, imported, stale, malformed, unavailable, and unknown", + "`evidence-receipt/v1`", + "`event/v1`", + "`usage-observation/v1`", + "`usage-estimate/v1`", + "`operator-report/v1`", + "`run-comparison/v1`", + "`redaction-policy/v1`", + "`cewp run verify`", "audit-only", - "App Server remains ungraduated", - "`codex-exec` fallback", - "external pilot evidence remains Phase 13 validation debt", - "No provider, desktop UI, terminal server, merge, push, publish, tag, or release automation", + "unavailable native usage remains unknown", + "clean Linux validation remains required", + "No publish, tag, or release", ]) { - assert(releaseNotes.slice(releaseIndex, previousIndex).includes(claim), `Phase 11 notes include honest claim: ${claim}`); + assert(releaseNotes.slice(releaseIndex, previousIndex).includes(claim), `Phase 12 notes include honest claim: ${claim}`); } - assert(packageJson.files.includes("docs/external-integration-boundary.md"), "Phase 11 boundary guide is in the package surface"); + assert(packageJson.files.includes("docs/evidence-receipts.md"), "Phase 12 evidence guide is in the package surface"); } try { runWorkflowReleaseContract(); - console.log("[PASS] Phase 11 version and release surface are aligned"); + console.log("[PASS] Phase 12 version and release surface are aligned"); } catch (error) { - console.error("[FAIL] Phase 11 release surface contract"); + console.error("[FAIL] Phase 12 release surface contract"); console.error(error && error.stack ? error.stack : error); process.exitCode = 1; } diff --git a/tests/harness/run-smoke.js b/tests/harness/run-smoke.js index c442adb..2314f97 100644 --- a/tests/harness/run-smoke.js +++ b/tests/harness/run-smoke.js @@ -2572,7 +2572,7 @@ async function main() { const pack = run("npm", ["pack", "--dry-run"], { cwd: cewpRoot, timeout: 120000 }); const packOutput = `${pack.stdout}\n${pack.stderr}`; assertExit(pack, 0, "npm pack --dry-run"); - assert(packageJson.version === "0.11.0-beta.0", `unexpected package version: ${packageJson.version}`); + assert(packageJson.version === "0.12.0-beta.0", `unexpected package version: ${packageJson.version}`); assert(packOutput.includes("docs/adapter-contract.md"), "adapter contract doc should be packed"); assert(packOutput.includes("docs/supervised-workflow.md"), "supervised workflow doc should be packed"); assert(packOutput.includes("docs/known-limitations.md"), "known limitations should be packed"); From dbd5a9fc58da4b4c946a32d53f0ababaec329ad6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tuncay=20=C3=96lmez?= Date: Fri, 24 Jul 2026 12:00:00 +0300 Subject: [PATCH 27/40] Design local Phase 13 pilot evidence --- ...26-07-22-phase-13-pilot-evidence-design.md | 151 ++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100644 docs/plans/2026-07-22-phase-13-pilot-evidence-design.md diff --git a/docs/plans/2026-07-22-phase-13-pilot-evidence-design.md b/docs/plans/2026-07-22-phase-13-pilot-evidence-design.md new file mode 100644 index 0000000..731c6f9 --- /dev/null +++ b/docs/plans/2026-07-22-phase-13-pilot-evidence-design.md @@ -0,0 +1,151 @@ +# Phase 13 Pilot Evidence Design + +Status: approved 2026-07-22 + +## Purpose + +Phase 13 must make real pilot evidence reviewable without inventing external +participants or adding mandatory telemetry. CEWP will provide a local, +machine-validated pilot ledger and privacy-safe export path. Maintainer +dogfooding remains useful engineering evidence but never satisfies an +independent-user gate. + +## Chosen Approach + +Pilot records live under the ignored runtime root: + +```text +.cewp/pilots//record.json +``` + +The alternatives were tracked public records by default and a documentation-only +process. Public-by-default records create unnecessary privacy risk. A +documentation-only process cannot enforce counting rules or prove which Phase 13 +gates remain open. A hosted service was rejected because the product is +local-first and does not require telemetry or an account. + +Only an explicit redacted export may leave the local pilot root. Export never +modifies the canonical record and never treats pattern redaction as proof that +arbitrary prose is secret-free. + +## Contracts + +`pilot-record/v1` is the canonical local record. It contains: + +- a stable pilot id and timestamps; +- participant classification: `maintainer-dogfood` or `independent-external`; +- a privacy-safe participant id supplied by the operator; +- repository attempt metadata using language, size, operating-system, test-stack, + input, risk, and mode buckets rather than an absolute path; +- the bounded task and golden-path outcomes; +- optional CEWP workflow run and receipt-integrity references; +- review, repeat-use, recovery, budget-pause, and controlled-host-limit outcomes; +- native-goal comparison evidence where genuinely comparable; +- onboarding failures, remediation, contributor feedback, and case-study status; +- provenance and warnings for evidence that is missing, malformed, or imported. + +`pilot-status/v1` is a derived read model. It reports every Phase 13 gate +separately, its threshold, qualifying evidence ids, exclusions, and remaining +count. It cannot mutate records. + +`pilot-export/v1` is a separately written redacted projection. It excludes raw +prompts, source code, logs, credentials, absolute paths, authentication material, +and unbounded free-form participant data. + +## CLI Surface + +The public commands are: + +```text +cewp pilot create +cewp pilot record +cewp pilot status +cewp pilot export +``` + +`create` writes one validated local skeleton. `record` applies an explicit, +validated observation without silently changing participant classification. +`status` evaluates the complete roadmap gate set and exits nonzero while any +required gate is unmet or invalid. `export` writes a redacted report only after +validation and discloses its redaction policy and counts. + +The first vertical slice is `create` plus `status`: a maintainer record can be +created and inspected, but every independent-user count remains zero. Later +slices add observation intake, linked receipt validation, recovery/comparison +evidence, and export. + +## Counting And Truth Rules + +- Only `independent-external` participants count toward external-participant, + full-reviewed-run, repeat-user, independent-repository, contribution, and case + study gates where the roadmap requires independent evidence. +- Multiple machines, repositories, or records belonging to the maintainer do not + become independent participants. +- A repository attempt is counted once by its stable attempt id. Duplicate ids, + conflicting participant classifications, or invalid timestamps fail closed. +- A full reviewed run requires linked final evidence and reviewer PASS; a test or + manually checked box alone is insufficient. +- Native-goal comparisons require equivalent task-shape metadata. Unavailable + usage remains unknown and is excluded from numeric deltas. +- Recovery, budget exhaustion, and host-limit scenarios are distinct evidence + categories and cannot substitute for one another. +- No status can report Phase 13 complete unless every roadmap gate has qualifying + evidence and no unresolved guardrail bypass is recorded. + +## Data Flow And Security + +The CLI resolves the repository root, then reads or atomically writes only below +`.cewp/pilots`. Pilot ids and export targets are validated against traversal, +absolute paths, reserved names, and symlink escapes. Canonical records use +bounded enums and structured fields instead of arbitrary prose wherever +possible. + +Run references are read-only. When a local workflow receipt is linked, CEWP +checks its run identity and integrity metadata; it does not copy raw evidence +files into the pilot record. Missing or stale references remain explicit and do +not qualify a gate. + +Export reuses the established redaction boundary, writes repository-relative +artifacts, and requires an operator-selected destination outside the ignored +canonical pilot directory only when the destination is safe. No command sends +data over the network. + +## Public Pilot And Contributor Surface + +Tracked repository assets will include: + +- issue forms for setup failure, workflow failure, feature request, and receipt + quality; +- a public case-study template with limitations and usage-truth fields; +- updated pilot instructions for manual, privacy-safe adoption milestones; +- contribution guidance, a concise architecture map, contract-extension + examples, and a private security-reporting path; +- explicit language that ecosystem listing and 1.0 remain gated on real external + evidence. + +Issue forms and templates collect no secrets, raw logs, repository paths, or +mandatory telemetry. + +## Error Handling + +Malformed JSON, unsupported schema versions, unsafe paths, duplicate identities, +contradictory observations, invalid gate claims, or broken receipt links produce +stable actionable errors. Read-only status remains available where possible and +marks affected evidence invalid rather than discarding it or converting it to +zero. Mutations use atomic writes and preserve the last valid record on failure. + +## Testing + +Development follows vertical TDD through the public CLI and Core modules: + +1. maintainer creation and an honestly incomplete status; +2. independent participant and repository-attempt counting; +3. reviewed run, repeat use, comparison, recovery, budget, host-limit, onboarding, + contribution, guardrail, and estimate-calibration gates; +4. duplicate, malformed, stale, traversal, symlink, and contradictory evidence; +5. redacted export and adversarial secret/path content; +6. package, help, documentation, Windows, and Linux contract coverage. + +Fixtures may prove behavior but never satisfy live pilot gates. The final Phase 13 +technical report will therefore distinguish implemented infrastructure from +external evidence still awaiting real users. From 0be69e034061c882c3e5e6734b50cbf9fa8bf843 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tuncay=20=C3=96lmez?= Date: Fri, 24 Jul 2026 12:30:00 +0300 Subject: [PATCH 28/40] Plan Phase 13 pilot evidence delivery --- ...-phase-13-pilot-evidence-implementation.md | 186 ++++++++++++++++++ 1 file changed, 186 insertions(+) create mode 100644 docs/plans/2026-07-22-phase-13-pilot-evidence-implementation.md diff --git a/docs/plans/2026-07-22-phase-13-pilot-evidence-implementation.md b/docs/plans/2026-07-22-phase-13-pilot-evidence-implementation.md new file mode 100644 index 0000000..e284c74 --- /dev/null +++ b/docs/plans/2026-07-22-phase-13-pilot-evidence-implementation.md @@ -0,0 +1,186 @@ +# Phase 13 Pilot Evidence Implementation Plan + +Status: approved design implementation + +## Constraints + +- Keep canonical pilot data under ignored `.cewp/pilots` runtime state. +- Never count fixtures or `maintainer-dogfood` as independent evidence. +- Do not add telemetry, a hosted service, another provider, or automatic remote + actions. +- Use public CLI behavior for contract tests and Core functions for validation. +- Keep the Phase 13 release gate incomplete until genuine external evidence + proves every threshold. +- Run the private-file tracking scan before every commit. + +## Slice 1: Honest Local Pilot Creation And Status + +Files: + +- `tests/contracts/pilot-record.js` +- `src/pilot/record.js` +- `src/pilot/status.js` +- `src/pilot/cli.js` +- `src/cli/parse.js` +- `src/cli/usage.js` +- `bin/cewp.js` +- `package.json` + +TDD behavior: + +1. `cewp pilot create --participant maintainer-dogfood --participant-id + --json` writes `pilot-record/v1` below `.cewp/pilots`. +2. `cewp pilot status --json` returns `pilot-status/v1`, reports the maintainer + record separately, and leaves every independent-user threshold at zero. +3. Status exits nonzero while Phase 13 gates are unmet but still emits valid JSON. +4. Unsafe ids, missing Git repository roots, duplicate ids, and unsupported + participant classifications fail with actionable errors. + +Focused verification: `npm run test:pilot-record`. + +## Slice 2: Structured Observation Intake And Gate Evaluation + +Files: + +- `tests/contracts/pilot-gates.js` +- `src/pilot/record.js` +- `src/pilot/status.js` +- `src/pilot/cli.js` +- `src/cli/parse.js` +- `package.json` + +TDD behavior: + +1. `cewp pilot record --from --yes --json` + atomically appends a validated observation. +2. Attempts, full reviewed runs, repeat use, native comparisons, recovery, + operational-budget exhaustion, host limits, onboarding remediation, + contribution, guardrail, case-study, and calibration evidence are counted only + when their required structured facts exist. +3. Duplicate attempt ids, participant reclassification, contradictory outcomes, + invalid timestamps, and impossible PASS claims fail closed. +4. Every roadmap gate exposes threshold, qualifying evidence, excluded evidence, + status, and remaining count. + +Focused verification: `npm run test:pilot-gates`. + +## Slice 3: Workflow Receipt Linking + +Files: + +- `tests/contracts/pilot-receipt-link.js` +- `src/pilot/evidence.js` +- `src/pilot/record.js` +- `src/pilot/status.js` +- `package.json` + +TDD behavior: + +1. Repository attempts may link a local workflow run without copying raw evidence. +2. A full reviewed run qualifies only when the run identity, final state, receipt + inventory, required verification, and reviewer PASS agree. +3. Missing, stale, partial, malformed, or integrity-failing receipt links are + warnings and never qualify a gate. +4. Native-only evidence remains explicitly imported and unknown usage never + becomes zero. + +Focused verification: `npm run test:pilot-receipt-link`. + +## Slice 4: Redacted Pilot Export + +Files: + +- `tests/contracts/pilot-export.js` +- `src/pilot/export.js` +- `src/pilot/cli.js` +- `src/cli/parse.js` +- `package.json` + +TDD behavior: + +1. `cewp pilot export [] --json` writes a separate + `pilot-export/v1` JSON/Markdown projection. +2. Canonical records are not modified and export is never implicit. +3. Absolute paths, traversal, secrets, credentials, raw prompts/logs/source, and + active markup are excluded or redacted with class/count disclosure. +4. Unsafe output targets and symlink escapes fail closed. + +Focused verification: `npm run test:pilot-export`. + +## Slice 5: Public Pilot Feedback And Case-Study Surface + +Files: + +- `.github/ISSUE_TEMPLATE/setup-failure.yml` +- `.github/ISSUE_TEMPLATE/workflow-failure.yml` +- `.github/ISSUE_TEMPLATE/feature-request.yml` +- `.github/ISSUE_TEMPLATE/receipt-quality.yml` +- `.github/ISSUE_TEMPLATE/config.yml` +- `docs/case-study-template.md` +- `docs/pilot-kit.md` +- `tests/contracts/pilot-public-surface.js` +- `package.json` + +TDD behavior: + +1. Four issue forms request sanitized reproducible evidence without secrets or + mandatory telemetry. +2. The case-study template covers task shape, plan quality, checkpoints, overhead, + truth labels, estimates, interventions, failures, receipt excerpt, and limits. +3. Pilot documentation explains local records, redacted export, honest dogfood, + and every external gate. + +Focused verification: `npm run test:pilot-public-surface`. + +## Slice 6: Contributor And Security Surface + +Files: + +- `CONTRIBUTING.md` +- `SECURITY.md` +- `docs/architecture.md` +- `docs/contract-extension-example.md` +- `tests/contracts/contributor-surface.js` +- `README.md` +- `package.json` + +TDD behavior: + +1. Contributors have local setup, focused tests, architecture boundaries, + contract-extension examples, issue selection guidance, and privacy rules. +2. Security reports use a private path and public issues explicitly reject + secrets and unredacted vulnerabilities. +3. Architecture retains Core authority, one execution owner/backend, and + provider-neutral contracts. + +Focused verification: `npm run test:contributor-surface`. + +## Slice 7: Phase 13 Release Surface + +Files: + +- `tests/contracts/pilot-release.js` +- `docs/release-notes.md` +- `docs/known-limitations.md` +- `README.md` +- `package.json` +- `plugins/cewp/.codex-plugin/plugin.json` + +TDD behavior: + +1. Prepare `0.13.0-beta.0` locally without publish, tag, or release. +2. Package the reviewed public pilot/contributor files but never `.cewp/pilots`, + private roadmap files, or `.cewp-private`. +3. Release notes distinguish infrastructure readiness, maintainer dogfood, and + missing external evidence. +4. Ecosystem submission remains unperformed until real case studies and golden + path evidence exist. + +Focused verification: `npm run test:pilot-release` and package dry runs. + +## Batch And Commit Boundaries + +Implement and commit in reviewable slices: local ledger/status; observations and +receipt links; redacted export; public pilot/contributor surface; Phase 13 release +preparation. Run focused tests during each red-green-refactor loop, then the full +baseline and privacy scan at the phase checkpoint. From 4aa87e437b92025c749c72fcdb3eb7a9269a0593 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tuncay=20=C3=96lmez?= Date: Fri, 24 Jul 2026 13:00:00 +0300 Subject: [PATCH 29/40] Add honest local pilot status --- bin/cewp.js | 8 ++- package.json | 7 ++- src/cli/parse.js | 31 ++++++++++- src/cli/usage.js | 2 + src/pilot/cli.js | 50 +++++++++++++++++ src/pilot/record.js | 76 ++++++++++++++++++++++++++ src/pilot/status.js | 97 +++++++++++++++++++++++++++++++++ tests/contracts/pilot-record.js | 87 +++++++++++++++++++++++++++++ 8 files changed, 353 insertions(+), 5 deletions(-) create mode 100644 src/pilot/cli.js create mode 100644 src/pilot/record.js create mode 100644 src/pilot/status.js create mode 100644 tests/contracts/pilot-record.js diff --git a/bin/cewp.js b/bin/cewp.js index 63325e2..3ca98cd 100644 --- a/bin/cewp.js +++ b/bin/cewp.js @@ -36,6 +36,7 @@ const { list, doctor } = require("../src/skills/status"); const { runSupervise } = require("../src/supervise/cli"); const { runWorkflow } = require("../src/workflow/cli"); const { runIntegration } = require("../src/integration/cli"); +const { runPilot } = require("../src/pilot/cli"); const { printHuman: printSupervisedDemo, runSupervisedDemo } = require("../src/demo/supervised"); function runDemo(options) { @@ -221,12 +222,17 @@ async function main() { return; } + if (args.command === "pilot") { + runPilot(args); + return; + } + if (args.command === "demo") { runDemo(args); return; } - if (!["init", "list", "doctor", "policy", "run", "supervise", "workflow", "integration", "demo"].includes(args.command)) { + if (!["init", "list", "doctor", "policy", "run", "supervise", "workflow", "integration", "pilot", "demo"].includes(args.command)) { throw new Error(`Unsupported command: ${args.command}`); } } catch (error) { diff --git a/package.json b/package.json index 01fd252..74248e5 100644 --- a/package.json +++ b/package.json @@ -52,8 +52,9 @@ "node": ">=22" }, "scripts": { - "test": "npm run test:contracts && npm run test:phase12 && npm run smoke", + "test": "npm run test:contracts && npm run test:phase12 && npm run test:phase13 && npm run smoke", "test:phase12": "npm run test:evidence-receipt && npm run test:event-run-verify && npm run test:operator-report && npm run test:run-comparison && npm run test:evidence-redaction", + "test:phase13": "npm run test:pilot-record", "test:contracts": "npm run test:init-install && npm run test:adapter-profile && npm run test:doctor-json && npm run test:operator-json && npm run test:hook-output && npm run test:skill-format && npm run test:ownership-gates && npm run test:fixtures && npm run test:plugin-package && npm run test:supervised-intake && npm run test:supervised-proposal && npm run test:supervised-controls && npm run test:supervised-execution && npm run test:supervised-review && npm run test:supervised-failure && npm run test:supervised-linear-resume && npm run test:supervised-demo && npm run test:workflow-definition && npm run test:workflow-compiler && npm run test:workflow-proposal && npm run test:workflow-scheduler && npm run test:workflow-worker-matrix && npm run test:workflow-result && npm run test:workflow-failure-result && npm run test:workflow-state-machine && npm run test:workflow-interventions && npm run test:workflow-failure-matrix && npm run test:workflow-budget && npm run test:workflow-progress && npm run test:workflow-review && npm run test:workflow-checkpoint-review && npm run test:workflow-lifecycle && npm run test:workflow-revision && npm run test:workflow-migration && npm run test:workflow-templates && npm run test:workflow-docs && npm run test:workflow-release && npm run test:integration-capabilities && npm run test:integration-binding && npm run test:integration-effort-policy && npm run test:integration-hook-evidence && npm run test:integration-mcp && npm run test:integration-observation && npm run test:native-goal-events", "test:init-install": "node tests/contracts/init-install.js", "test:adapter-profile": "node tests/contracts/adapter-profile.js", @@ -104,6 +105,7 @@ "test:evidence-redaction": "node tests/contracts/evidence-redaction.js", "test:integration-observation": "node tests/contracts/integration-observation.js", "test:native-goal-events": "node tests/contracts/native-goal-events.js", + "test:pilot-record": "node tests/contracts/pilot-record.js", "demo:supervised": "node src/demo/supervised.js", "test:plugin-lifecycle": "node tests/capabilities/plugin-lifecycle.js", "test:clean-install": "node tests/capabilities/clean-install.js", @@ -114,7 +116,8 @@ "check:workflow-syntax": "node --check ./src/workflow/budget.js && node --check ./src/workflow/compiler.js && node --check ./src/workflow/migration.js && node --check ./src/workflow/progress.js && node --check ./src/workflow/proposal.js && node --check ./src/workflow/review.js && node --check ./src/workflow/revision.js && node --check ./src/workflow/templates.js && node --check ./tests/contracts/workflow-budget.js && node --check ./tests/contracts/workflow-checkpoint-review.js && node --check ./tests/contracts/workflow-compiler.js && node --check ./tests/contracts/workflow-docs.js && node --check ./tests/contracts/workflow-failure-matrix.js && node --check ./tests/contracts/workflow-failure-result.js && node --check ./tests/contracts/workflow-migration.js && node --check ./tests/contracts/workflow-progress.js && node --check ./tests/contracts/workflow-release.js && node --check ./tests/contracts/workflow-review.js && node --check ./tests/contracts/workflow-lifecycle.js && node --check ./tests/contracts/workflow-revision.js && node --check ./tests/contracts/workflow-templates.js && node --check ./tests/contracts/workflow-worker-matrix.js", "check:mcp": "node --check ./bin/cewp-mcp.js && node --check ./src/mcp/server.js && node --check ./src/mcp/tools.js && node --check ./tests/contracts/integration-mcp.js", "check:evidence": "node --check ./src/evidence/receipt.js && node --check ./src/evidence/events.js && node --check ./src/evidence/report.js && node --check ./src/evidence/compare.js && node --check ./src/evidence/redaction.js && node --check ./src/evidence/usage.js && node --check ./src/evidence/verify.js && node --check ./src/run/verify.js && node --check ./tests/contracts/evidence-receipt.js && node --check ./tests/contracts/event-run-verify.js && node --check ./tests/contracts/operator-report.js && node --check ./tests/contracts/run-comparison.js && node --check ./tests/contracts/evidence-redaction.js", - "check": "npm run check:syntax && npm run check:workflow-syntax && npm run check:mcp && npm run check:evidence && npm test", + "check:pilot": "node --check ./src/pilot/record.js && node --check ./src/pilot/status.js && node --check ./src/pilot/cli.js && node --check ./tests/contracts/pilot-record.js", + "check": "npm run check:syntax && npm run check:workflow-syntax && npm run check:mcp && npm run check:evidence && npm run check:pilot && npm test", "pack:dry-run": "npm pack --dry-run" }, "keywords": [ diff --git a/src/cli/parse.js b/src/cli/parse.js index 64a73fc..b1e16c1 100644 --- a/src/cli/parse.js +++ b/src/cli/parse.js @@ -79,6 +79,9 @@ function parseArgs(argv) { model: undefined, effort: undefined, comparisonRunId: undefined, + pilotId: undefined, + participant: undefined, + participantId: undefined, }; if (argv[0] === "--help" || argv[0] === "-h") { @@ -98,11 +101,35 @@ function parseArgs(argv) { return args; } - const optionStart = ["run", "supervise", "workflow", "integration", "demo"].includes(args.command) ? 2 : 1; + const optionStart = ["run", "supervise", "workflow", "integration", "pilot", "demo"].includes(args.command) ? 2 : 1; for (let index = optionStart; index < argv.length; index += 1) { const arg = argv[index]; + if (args.command === "pilot" && arg === "--pilot-id") { + const value = argv[index + 1]; + if (!value || value.startsWith("--")) throw new Error("--pilot-id requires an id."); + args.pilotId = value; + index += 1; + continue; + } + + if (args.command === "pilot" && arg === "--participant") { + const value = argv[index + 1]; + if (!value || value.startsWith("--")) throw new Error("--participant requires a classification."); + args.participant = value; + index += 1; + continue; + } + + if (args.command === "pilot" && arg === "--participant-id") { + const value = argv[index + 1]; + if (!value || value.startsWith("--")) throw new Error("--participant-id requires an id."); + args.participantId = value; + index += 1; + continue; + } + if (args.command === "run" && args.subcommand === "prompt" && index === 2) { args.role = arg; continue; @@ -472,7 +499,7 @@ function parseArgs(argv) { continue; } - if (["doctor", "run", "supervise", "workflow", "integration", "demo"].includes(args.command) && arg === "--json") { + if (["doctor", "run", "supervise", "workflow", "integration", "pilot", "demo"].includes(args.command) && arg === "--json") { args.json = true; continue; } diff --git a/src/cli/usage.js b/src/cli/usage.js index 8b3eaf6..715e027 100644 --- a/src/cli/usage.js +++ b/src/cli/usage.js @@ -7,6 +7,8 @@ Usage: cewp init [--mode repo|global] [--target ] [--force] [--with-config] cewp list [--mode repo|global] [--target ] cewp doctor [--mode repo|global] [--target ] [--json] + cewp pilot create --pilot-id --participant --participant-id [--json] + cewp pilot status [--json] cewp workflow validate [--json] cewp workflow compile (--goal | --from ) [--source-kind ] [--json] cewp workflow template [--json] diff --git a/src/pilot/cli.js b/src/pilot/cli.js new file mode 100644 index 0000000..985a2c9 --- /dev/null +++ b/src/pilot/cli.js @@ -0,0 +1,50 @@ +"use strict"; + +const { createPilotRecord } = require("./record"); +const { derivePilotStatus } = require("./status"); + +function outputJson(command, data) { + console.log(JSON.stringify({ + schemaVersion: "operator-json/v1", + command, + generatedAt: new Date().toISOString(), + data, + warnings: [], + }, null, 2)); +} + +function runPilot(options = {}) { + if (options.subcommand === "status") { + const status = derivePilotStatus(process.cwd()); + if (options.json) outputJson("pilot.status", status); + else { + console.log("CEWP Phase 13 pilot status"); + console.log(`Complete: ${status.complete ? "yes" : "no"}`); + console.log(`Independent participants: ${status.participants.independentExternal}/3`); + for (const gate of status.gates) { + console.log(`${gate.id}: ${gate.observed}/${gate.threshold} (${gate.status})`); + } + } + if (!status.complete) process.exitCode = 1; + return; + } + if (options.subcommand !== "create") { + throw new Error(`Unsupported pilot command: ${options.subcommand || "missing"}.`); + } + const record = createPilotRecord({ + repoRoot: process.cwd(), + pilotId: options.pilotId, + participant: options.participant, + participantId: options.participantId, + }); + if (options.json) outputJson("pilot.create", record); + else { + console.log("CEWP local pilot record created"); + console.log(`Pilot ID: ${record.pilotId}`); + console.log(`Participant: ${record.participant.classification}`); + } +} + +module.exports = { + runPilot, +}; diff --git a/src/pilot/record.js b/src/pilot/record.js new file mode 100644 index 0000000..4a04559 --- /dev/null +++ b/src/pilot/record.js @@ -0,0 +1,76 @@ +"use strict"; + +const fs = require("node:fs"); +const path = require("node:path"); +const { getGitHeadCommit } = require("../lib/git"); +const { writeJsonAtomic } = require("../workflow/state"); + +const PILOT_RECORD_SCHEMA_VERSION = "pilot-record/v1"; +const PARTICIPANT_CLASSIFICATIONS = new Set([ + "maintainer-dogfood", + "independent-external", +]); +const SAFE_ID = /^[a-z0-9][a-z0-9-]{0,63}$/; + +function validatePilotId(value) { + const pilotId = String(value || "").trim(); + if (!SAFE_ID.test(pilotId)) { + throw new Error("--pilot-id requires 1-64 lowercase letters, numbers, or hyphens."); + } + return pilotId; +} + +function validateParticipantId(value) { + const participantId = String(value || "").trim(); + if (!SAFE_ID.test(participantId)) { + throw new Error("--participant-id requires a privacy-safe lowercase id."); + } + return participantId; +} + +function validateParticipantClassification(value) { + const classification = String(value || "").trim(); + if (!PARTICIPANT_CLASSIFICATIONS.has(classification)) { + throw new Error("--participant requires maintainer-dogfood or independent-external."); + } + return classification; +} + +function createPilotRecord(options = {}) { + const repoRoot = path.resolve(options.repoRoot || process.cwd()); + const pilotId = validatePilotId(options.pilotId); + const participantId = validateParticipantId(options.participantId); + const classification = validateParticipantClassification(options.participant); + const recordPath = path.join(repoRoot, ".cewp", "pilots", pilotId, "record.json"); + if (fs.existsSync(recordPath)) { + throw new Error(`Pilot record already exists: ${pilotId}.`); + } + const timestamp = (options.now || new Date()).toISOString(); + const record = { + schemaVersion: PILOT_RECORD_SCHEMA_VERSION, + pilotId, + participant: { + classification, + id: participantId, + independentEvidenceEligible: classification === "independent-external", + }, + repository: { + gitBaseCommit: getGitHeadCommit(repoRoot), + pathRecorded: false, + }, + observations: [], + createdAt: timestamp, + updatedAt: timestamp, + }; + writeJsonAtomic(recordPath, record); + return record; +} + +module.exports = { + PARTICIPANT_CLASSIFICATIONS, + PILOT_RECORD_SCHEMA_VERSION, + createPilotRecord, + validateParticipantClassification, + validateParticipantId, + validatePilotId, +}; diff --git a/src/pilot/status.js b/src/pilot/status.js new file mode 100644 index 0000000..35af0a8 --- /dev/null +++ b/src/pilot/status.js @@ -0,0 +1,97 @@ +"use strict"; + +const fs = require("node:fs"); +const path = require("node:path"); +const { PILOT_RECORD_SCHEMA_VERSION } = require("./record"); + +const PILOT_STATUS_SCHEMA_VERSION = "pilot-status/v1"; +const COUNT_GATES = Object.freeze([ + { id: "independent-repository-attempts", threshold: 10, observationType: "repository-attempt" }, + { id: "independent-external-participants", threshold: 3, source: "participants" }, + { id: "real-bounded-external-repository-task", threshold: 1, observationType: "bounded-external-task" }, + { id: "full-reviewed-runs", threshold: 5, observationType: "full-reviewed-run" }, + { id: "repeat-users-without-maintainer-assistance", threshold: 3, observationType: "repeat-user" }, + { id: "comparable-native-goal-runs", threshold: 3, observationType: "native-goal-comparison" }, + { id: "measurable-cewp-benefit", threshold: 1, observationType: "measurable-benefit" }, + { id: "recovered-pause-or-failure-scenarios", threshold: 3, observationType: "recovery" }, + { id: "operational-budget-exhaustion", threshold: 1, observationType: "operational-budget-exhaustion" }, + { id: "controlled-host-limit", threshold: 1, observationType: "controlled-host-limit" }, + { id: "top-onboarding-failures-remediated", threshold: 5, observationType: "onboarding-remediation" }, + { id: "external-contribution-or-substantive-issue", threshold: 1, observationType: "external-contribution" }, + { id: "public-case-studies", threshold: 3, observationType: "public-case-study" }, + { id: "usage-estimate-calibration-reported", threshold: 1, observationType: "estimate-calibration-report" }, + { id: "guardrail-audit-with-no-unresolved-bypass", threshold: 1, observationType: "guardrail-audit-pass" }, +]); + +function loadPilotRecords(repoRoot) { + const pilotsRoot = path.join(path.resolve(repoRoot), ".cewp", "pilots"); + if (!fs.existsSync(pilotsRoot)) return []; + return fs.readdirSync(pilotsRoot, { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .sort((left, right) => left.name.localeCompare(right.name)) + .map((entry) => { + const filePath = path.join(pilotsRoot, entry.name, "record.json"); + if (!fs.existsSync(filePath)) return null; + const record = JSON.parse(fs.readFileSync(filePath, "utf8")); + if (record.schemaVersion !== PILOT_RECORD_SCHEMA_VERSION) { + throw new Error(`Unsupported pilot record schema for ${entry.name}: ${record.schemaVersion || "missing"}.`); + } + return record; + }) + .filter(Boolean); +} + +function derivePilotStatus(repoRoot) { + const records = loadPilotRecords(repoRoot); + const externalRecords = records.filter((record) => record.participant.classification === "independent-external"); + const externalParticipantIds = new Set(externalRecords.map((record) => record.participant.id)); + const evidenceByType = new Map(); + for (const record of externalRecords) { + for (const observation of record.observations || []) { + if (!observation || observation.qualifies !== true || typeof observation.type !== "string") continue; + if (!evidenceByType.has(observation.type)) evidenceByType.set(observation.type, []); + evidenceByType.get(observation.type).push(`${record.pilotId}:${observation.id || observation.type}`); + } + } + const gates = COUNT_GATES.map((definition) => { + const qualifyingEvidence = definition.source === "participants" + ? [...externalParticipantIds].sort() + : [...new Set(evidenceByType.get(definition.observationType) || [])].sort(); + const observed = qualifyingEvidence.length; + return { + id: definition.id, + threshold: definition.threshold, + observed, + remaining: Math.max(0, definition.threshold - observed), + status: observed >= definition.threshold ? "met" : "unmet", + qualifyingEvidence, + }; + }); + return { + schemaVersion: PILOT_STATUS_SCHEMA_VERSION, + complete: gates.every((gate) => gate.status === "met"), + records: { total: records.length, valid: records.length, invalid: 0 }, + participants: { + maintainerDogfood: records.filter((record) => record.participant.classification === "maintainer-dogfood").length, + independentExternal: externalParticipantIds.size, + }, + gates, + exclusions: records + .filter((record) => record.participant.classification !== "independent-external") + .map((record) => ({ + pilotId: record.pilotId, + classification: record.participant.classification, + reason: "maintainer dogfood never counts as independent Phase 13 evidence", + })), + warnings: gates + .filter((gate) => gate.status === "unmet") + .map((gate) => ({ code: "pilot-gate-unmet", gate: gate.id, remaining: gate.remaining })), + }; +} + +module.exports = { + COUNT_GATES, + PILOT_STATUS_SCHEMA_VERSION, + derivePilotStatus, + loadPilotRecords, +}; diff --git a/tests/contracts/pilot-record.js b/tests/contracts/pilot-record.js new file mode 100644 index 0000000..43e5aef --- /dev/null +++ b/tests/contracts/pilot-record.js @@ -0,0 +1,87 @@ +"use strict"; + +const fs = require("node:fs"); +const path = require("node:path"); +const { assert } = require("../harness/lib/assertions"); +const { cleanupRepo, makeTempRepo, readJson, runNode } = require("../harness/lib/temp-repo"); + +const cewpCli = path.join(__dirname, "..", "..", "bin", "cewp.js"); + +function runContract() { + const repoRoot = makeTempRepo("cewp-pilot-record-"); + try { + const created = runNode(cewpCli, [ + "pilot", "create", + "--pilot-id", "dogfood-1", + "--participant", "maintainer-dogfood", + "--participant-id", "maintainer-1", + "--json", + ], repoRoot); + assert(created.status === 0, `pilot create succeeds: ${created.stderr}`); + const output = JSON.parse(created.stdout); + assert(output.command === "pilot.create", "pilot create has a stable command envelope"); + assert(output.data.schemaVersion === "pilot-record/v1", "pilot record is versioned"); + assert(output.data.pilotId === "dogfood-1", "pilot id is explicit and stable"); + assert(output.data.participant.classification === "maintainer-dogfood", "maintainer evidence is classified honestly"); + assert(output.data.participant.id === "maintainer-1", "privacy-safe participant id is retained"); + + const recordPath = path.join(repoRoot, ".cewp", "pilots", "dogfood-1", "record.json"); + assert(fs.existsSync(recordPath), "canonical pilot record stays under ignored local runtime state"); + assert(readJson(recordPath).pilotId === "dogfood-1", "persisted pilot record matches CLI output"); + + const statusResult = runNode(cewpCli, ["pilot", "status", "--json"], repoRoot); + assert(statusResult.status === 1, "incomplete Phase 13 status exits nonzero"); + const status = JSON.parse(statusResult.stdout); + assert(status.command === "pilot.status", "pilot status has a stable command envelope"); + assert(status.data.schemaVersion === "pilot-status/v1", "pilot status is versioned"); + assert(status.data.complete === false, "maintainer dogfood cannot complete Phase 13"); + assert(status.data.participants.maintainerDogfood === 1, "maintainer dogfood is visible"); + assert(status.data.participants.independentExternal === 0, "maintainer dogfood is excluded from independent participants"); + const externalGate = status.data.gates.find((gate) => gate.id === "independent-external-participants"); + const repositoryGate = status.data.gates.find((gate) => gate.id === "independent-repository-attempts"); + const reviewedGate = status.data.gates.find((gate) => gate.id === "full-reviewed-runs"); + assert(externalGate.threshold === 3 && externalGate.observed === 0 && externalGate.status === "unmet", "three independent participants remain required"); + assert(repositoryGate.threshold === 10 && repositoryGate.observed === 0, "ten repository attempts remain required"); + assert(reviewedGate.threshold === 5 && reviewedGate.observed === 0, "five reviewed runs remain required"); + assert(status.data.exclusions.some((entry) => entry.pilotId === "dogfood-1"), "excluded maintainer evidence is explained"); + + const duplicate = runNode(cewpCli, [ + "pilot", "create", + "--pilot-id", "dogfood-1", + "--participant", "maintainer-dogfood", + "--participant-id", "maintainer-1", + "--json", + ], repoRoot); + assert(duplicate.status === 1 && duplicate.stderr.includes("already exists"), "duplicate pilot ids fail closed"); + + const traversal = runNode(cewpCli, [ + "pilot", "create", + "--pilot-id", "../escape", + "--participant", "maintainer-dogfood", + "--participant-id", "maintainer-1", + "--json", + ], repoRoot); + assert(traversal.status === 1 && traversal.stderr.includes("--pilot-id"), "unsafe pilot ids are rejected before a path is created"); + + const fabricatedClass = runNode(cewpCli, [ + "pilot", "create", + "--pilot-id", "fake-external", + "--participant", "external-ish", + "--participant-id", "person-1", + "--json", + ], repoRoot); + assert(fabricatedClass.status === 1 && fabricatedClass.stderr.includes("independent-external"), "unsupported participant classifications fail closed"); + assert(!fs.existsSync(path.resolve(repoRoot, "..", "escape")), "unsafe pilot ids cannot escape the local pilot root"); + } finally { + cleanupRepo(repoRoot); + } +} + +try { + runContract(); + console.log("[PASS] local pilot record creation"); +} catch (error) { + console.error("[FAIL] local pilot record creation"); + console.error(error && error.stack ? error.stack : error); + process.exitCode = 1; +} From 586970609405d1fae049b3cf636ea7c090ca1408 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tuncay=20=C3=96lmez?= Date: Fri, 24 Jul 2026 13:30:00 +0300 Subject: [PATCH 30/40] Add structured pilot gate evidence --- package.json | 5 +- src/cli/parse.js | 12 +- src/cli/usage.js | 1 + src/pilot/cli.js | 17 ++ src/pilot/observation.js | 377 +++++++++++++++++++++++++++++++++ src/pilot/status.js | 8 +- tests/contracts/pilot-gates.js | 251 ++++++++++++++++++++++ 7 files changed, 666 insertions(+), 5 deletions(-) create mode 100644 src/pilot/observation.js create mode 100644 tests/contracts/pilot-gates.js diff --git a/package.json b/package.json index 74248e5..488662f 100644 --- a/package.json +++ b/package.json @@ -54,7 +54,7 @@ "scripts": { "test": "npm run test:contracts && npm run test:phase12 && npm run test:phase13 && npm run smoke", "test:phase12": "npm run test:evidence-receipt && npm run test:event-run-verify && npm run test:operator-report && npm run test:run-comparison && npm run test:evidence-redaction", - "test:phase13": "npm run test:pilot-record", + "test:phase13": "npm run test:pilot-record && npm run test:pilot-gates", "test:contracts": "npm run test:init-install && npm run test:adapter-profile && npm run test:doctor-json && npm run test:operator-json && npm run test:hook-output && npm run test:skill-format && npm run test:ownership-gates && npm run test:fixtures && npm run test:plugin-package && npm run test:supervised-intake && npm run test:supervised-proposal && npm run test:supervised-controls && npm run test:supervised-execution && npm run test:supervised-review && npm run test:supervised-failure && npm run test:supervised-linear-resume && npm run test:supervised-demo && npm run test:workflow-definition && npm run test:workflow-compiler && npm run test:workflow-proposal && npm run test:workflow-scheduler && npm run test:workflow-worker-matrix && npm run test:workflow-result && npm run test:workflow-failure-result && npm run test:workflow-state-machine && npm run test:workflow-interventions && npm run test:workflow-failure-matrix && npm run test:workflow-budget && npm run test:workflow-progress && npm run test:workflow-review && npm run test:workflow-checkpoint-review && npm run test:workflow-lifecycle && npm run test:workflow-revision && npm run test:workflow-migration && npm run test:workflow-templates && npm run test:workflow-docs && npm run test:workflow-release && npm run test:integration-capabilities && npm run test:integration-binding && npm run test:integration-effort-policy && npm run test:integration-hook-evidence && npm run test:integration-mcp && npm run test:integration-observation && npm run test:native-goal-events", "test:init-install": "node tests/contracts/init-install.js", "test:adapter-profile": "node tests/contracts/adapter-profile.js", @@ -106,6 +106,7 @@ "test:integration-observation": "node tests/contracts/integration-observation.js", "test:native-goal-events": "node tests/contracts/native-goal-events.js", "test:pilot-record": "node tests/contracts/pilot-record.js", + "test:pilot-gates": "node tests/contracts/pilot-gates.js", "demo:supervised": "node src/demo/supervised.js", "test:plugin-lifecycle": "node tests/capabilities/plugin-lifecycle.js", "test:clean-install": "node tests/capabilities/clean-install.js", @@ -116,7 +117,7 @@ "check:workflow-syntax": "node --check ./src/workflow/budget.js && node --check ./src/workflow/compiler.js && node --check ./src/workflow/migration.js && node --check ./src/workflow/progress.js && node --check ./src/workflow/proposal.js && node --check ./src/workflow/review.js && node --check ./src/workflow/revision.js && node --check ./src/workflow/templates.js && node --check ./tests/contracts/workflow-budget.js && node --check ./tests/contracts/workflow-checkpoint-review.js && node --check ./tests/contracts/workflow-compiler.js && node --check ./tests/contracts/workflow-docs.js && node --check ./tests/contracts/workflow-failure-matrix.js && node --check ./tests/contracts/workflow-failure-result.js && node --check ./tests/contracts/workflow-migration.js && node --check ./tests/contracts/workflow-progress.js && node --check ./tests/contracts/workflow-release.js && node --check ./tests/contracts/workflow-review.js && node --check ./tests/contracts/workflow-lifecycle.js && node --check ./tests/contracts/workflow-revision.js && node --check ./tests/contracts/workflow-templates.js && node --check ./tests/contracts/workflow-worker-matrix.js", "check:mcp": "node --check ./bin/cewp-mcp.js && node --check ./src/mcp/server.js && node --check ./src/mcp/tools.js && node --check ./tests/contracts/integration-mcp.js", "check:evidence": "node --check ./src/evidence/receipt.js && node --check ./src/evidence/events.js && node --check ./src/evidence/report.js && node --check ./src/evidence/compare.js && node --check ./src/evidence/redaction.js && node --check ./src/evidence/usage.js && node --check ./src/evidence/verify.js && node --check ./src/run/verify.js && node --check ./tests/contracts/evidence-receipt.js && node --check ./tests/contracts/event-run-verify.js && node --check ./tests/contracts/operator-report.js && node --check ./tests/contracts/run-comparison.js && node --check ./tests/contracts/evidence-redaction.js", - "check:pilot": "node --check ./src/pilot/record.js && node --check ./src/pilot/status.js && node --check ./src/pilot/cli.js && node --check ./tests/contracts/pilot-record.js", + "check:pilot": "node --check ./src/pilot/record.js && node --check ./src/pilot/observation.js && node --check ./src/pilot/status.js && node --check ./src/pilot/cli.js && node --check ./tests/contracts/pilot-record.js && node --check ./tests/contracts/pilot-gates.js", "check": "npm run check:syntax && npm run check:workflow-syntax && npm run check:mcp && npm run check:evidence && npm run check:pilot && npm test", "pack:dry-run": "npm pack --dry-run" }, diff --git a/src/cli/parse.js b/src/cli/parse.js index b1e16c1..be9599e 100644 --- a/src/cli/parse.js +++ b/src/cli/parse.js @@ -106,6 +106,11 @@ function parseArgs(argv) { for (let index = optionStart; index < argv.length; index += 1) { const arg = argv[index]; + if (args.command === "pilot" && args.subcommand === "record" && index === 2 && !arg.startsWith("--")) { + args.pilotId = arg; + continue; + } + if (args.command === "pilot" && arg === "--pilot-id") { const value = argv[index + 1]; if (!value || value.startsWith("--")) throw new Error("--pilot-id requires an id."); @@ -514,6 +519,11 @@ function parseArgs(argv) { continue; } + if (args.command === "pilot" && arg === "--yes") { + args.yes = true; + continue; + } + if (args.command === "integration" && arg === "--yes") { args.yes = true; @@ -573,7 +583,7 @@ function parseArgs(argv) { continue; } - if (["run", "supervise", "workflow"].includes(args.command) && arg === "--from") { + if (["run", "supervise", "workflow", "pilot"].includes(args.command) && arg === "--from") { const value = argv[index + 1]; if (!value || value.startsWith("--")) { throw new Error("--from requires a file path."); diff --git a/src/cli/usage.js b/src/cli/usage.js index 715e027..e4efe1c 100644 --- a/src/cli/usage.js +++ b/src/cli/usage.js @@ -8,6 +8,7 @@ Usage: cewp list [--mode repo|global] [--target ] cewp doctor [--mode repo|global] [--target ] [--json] cewp pilot create --pilot-id --participant --participant-id [--json] + cewp pilot record --from --yes [--json] cewp pilot status [--json] cewp workflow validate [--json] cewp workflow compile (--goal | --from ) [--source-kind ] [--json] diff --git a/src/pilot/cli.js b/src/pilot/cli.js index 985a2c9..baa1a61 100644 --- a/src/pilot/cli.js +++ b/src/pilot/cli.js @@ -2,6 +2,7 @@ const { createPilotRecord } = require("./record"); const { derivePilotStatus } = require("./status"); +const { recordPilotObservation } = require("./observation"); function outputJson(command, data) { console.log(JSON.stringify({ @@ -14,6 +15,22 @@ function outputJson(command, data) { } function runPilot(options = {}) { + if (options.subcommand === "record") { + if (!options.yes) throw new Error("pilot record requires --yes confirmation."); + const result = recordPilotObservation({ + repoRoot: process.cwd(), + pilotId: options.pilotId, + fromFile: options.fromFile, + }); + if (options.json) outputJson("pilot.record", result); + else { + console.log("CEWP pilot observation recorded"); + console.log(`Pilot ID: ${result.record.pilotId}`); + console.log(`Observation: ${result.observation.id} (${result.observation.type})`); + console.log(`Independent evidence eligible: ${result.observation.qualification.eligible ? "yes" : "no"}`); + } + return; + } if (options.subcommand === "status") { const status = derivePilotStatus(process.cwd()); if (options.json) outputJson("pilot.status", status); diff --git a/src/pilot/observation.js b/src/pilot/observation.js new file mode 100644 index 0000000..e5aa52b --- /dev/null +++ b/src/pilot/observation.js @@ -0,0 +1,377 @@ +"use strict"; + +const fs = require("node:fs"); +const path = require("node:path"); +const { readRepoJson } = require("../workflow/source"); +const { writeJsonAtomic } = require("../workflow/state"); +const { PILOT_RECORD_SCHEMA_VERSION, validatePilotId } = require("./record"); + +const PILOT_OBSERVATION_SCHEMA_VERSION = "pilot-observation/v1"; +const SAFE_ID = /^[a-z0-9][a-z0-9-]{0,63}$/; +const OBSERVATION_TYPES = new Set([ + "repository-attempt", + "golden-path-complete", + "bounded-external-task", + "repeat-user", + "native-goal-comparison", + "measurable-benefit", + "recovery", + "operational-budget-exhaustion", + "controlled-host-limit", + "onboarding-remediation", + "external-contribution", + "public-case-study", + "estimate-calibration-report", + "guardrail-audit-pass", +]); +const ATTEMPT_ENUMS = Object.freeze({ + sizeBucket: new Set(["small", "medium", "large", "unknown"]), + operatingSystem: new Set(["windows", "linux", "macos", "other"]), + inputKind: new Set(["direct-goal", "issue", "prd", "plan", "progress"]), + riskLevel: new Set(["low", "medium", "high"]), + mode: new Set(["supervised", "autonomous", "audit-only"]), +}); + +function requireSafeId(value, label) { + const normalized = String(value || "").trim(); + if (!SAFE_ID.test(normalized)) throw new Error(`${label} requires a privacy-safe lowercase id.`); + return normalized; +} + +function requireLabel(value, label) { + const normalized = String(value || "").trim().toLowerCase(); + if (!/^[a-z0-9][a-z0-9.+#-]{0,31}$/.test(normalized)) { + throw new Error(`${label} requires a bounded privacy-safe label.`); + } + return normalized; +} + +function requireEnum(value, name) { + if (!ATTEMPT_ENUMS[name].has(value)) { + throw new Error(`pilot observation attempt.${name} is unsupported.`); + } + return value; +} + +function normalizeObservedAt(value) { + if (typeof value !== "string" || !/^\d{4}-\d{2}-\d{2}T/.test(value) || Number.isNaN(Date.parse(value))) { + throw new Error("pilot observation observedAt requires an ISO-8601 timestamp."); + } + return new Date(value).toISOString(); +} + +function validateRepositoryAttempt(value) { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error("repository-attempt observation requires attempt metadata."); + } + return { + id: requireSafeId(value.id, "pilot observation attempt.id"), + language: requireLabel(value.language, "pilot observation attempt.language"), + sizeBucket: requireEnum(value.sizeBucket, "sizeBucket"), + operatingSystem: requireEnum(value.operatingSystem, "operatingSystem"), + testStack: requireLabel(value.testStack, "pilot observation attempt.testStack"), + inputKind: requireEnum(value.inputKind, "inputKind"), + riskLevel: requireEnum(value.riskLevel, "riskLevel"), + mode: requireEnum(value.mode, "mode"), + }; +} + +function validateGoldenPath(value) { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error("golden-path-complete observation requires completion metadata."); + } + if (value.supervised !== true || value.completed !== true || value.participantConfirmed !== true) { + throw new Error("golden-path-complete requires supervised, completed, and participantConfirmed to be true."); + } + if (!Number.isFinite(value.firstApprovalMinutes) || value.firstApprovalMinutes < 0 || value.firstApprovalMinutes > 300) { + throw new Error("golden-path-complete requires bounded firstApprovalMinutes."); + } + return { + supervised: true, + completed: true, + participantConfirmed: true, + firstApprovalMinutes: value.firstApprovalMinutes, + }; +} + +function validateBoundedExternalTask(value) { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error("bounded-external-task observation requires task metadata."); + } + if (value.bounded !== true || value.realRepository !== true || value.acceptanceDefinedBeforeExecution !== true) { + throw new Error("bounded-external-task requires bounded, realRepository, and acceptanceDefinedBeforeExecution to be true."); + } + return { + bounded: true, + realRepository: true, + acceptanceDefinedBeforeExecution: true, + }; +} + +function requireObject(value, label) { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error(`${label} requires structured metadata.`); + } + return value; +} + +function requireTrue(value, label) { + if (value !== true) throw new Error(`${label} must be true.`); + return true; +} + +function requireFalse(value, label) { + if (value !== false) throw new Error(`${label} must be false.`); + return false; +} + +function validateRepeatUse(value) { + const input = requireObject(value, "repeat-user"); + if (!Number.isInteger(input.runOrdinal) || input.runOrdinal < 2) { + throw new Error("repeat-user runOrdinal must be at least 2."); + } + return { + runOrdinal: input.runOrdinal, + withoutMaintainerAssistance: requireTrue(input.withoutMaintainerAssistance, "repeat-user withoutMaintainerAssistance"), + }; +} + +function validateComparison(value) { + const input = requireObject(value, "native-goal-comparison"); + const truthLabels = new Set(["observed", "unknown"]); + if (!truthLabels.has(input.nativeUsage) || !truthLabels.has(input.cewpUsage)) { + throw new Error("native-goal-comparison usage labels must be observed or unknown."); + } + return { + id: requireSafeId(input.id, "native-goal-comparison id"), + equivalentTaskShape: requireTrue(input.equivalentTaskShape, "native-goal-comparison equivalentTaskShape"), + setupOverheadMeasured: requireTrue(input.setupOverheadMeasured, "native-goal-comparison setupOverheadMeasured"), + nativeUsage: input.nativeUsage, + cewpUsage: input.cewpUsage, + }; +} + +function validateBenefit(value) { + const input = requireObject(value, "measurable-benefit"); + const dimensions = new Set(["recovery", "policy-enforcement", "independent-findings", "evidence-quality"]); + if (!dimensions.has(input.dimension)) throw new Error("measurable-benefit dimension is unsupported."); + return { + dimension: input.dimension, + measured: requireTrue(input.measured, "measurable-benefit measured"), + setupOverheadConsidered: requireTrue(input.setupOverheadConsidered, "measurable-benefit setupOverheadConsidered"), + }; +} + +function validateRecovery(value) { + const input = requireObject(value, "recovery"); + if (!["pause-revise-resume", "failure-retry"].includes(input.scenario)) { + throw new Error("recovery scenario is unsupported."); + } + return { + id: requireSafeId(input.id, "recovery id"), + scenario: input.scenario, + recoveredWithoutRestart: requireTrue(input.recoveredWithoutRestart, "recovery recoveredWithoutRestart"), + priorEvidenceRetained: requireTrue(input.priorEvidenceRetained, "recovery priorEvidenceRetained"), + }; +} + +function validatePause(value, expectedType) { + const input = requireObject(value, expectedType); + const allowedStates = expectedType === "controlled-host-limit" + ? ["paused-host-limit"] + : ["paused-budget-safe", "paused-budget-unverified"]; + if (!allowedStates.includes(input.state)) throw new Error(`${expectedType} pause state is unsupported.`); + return { + state: input.state, + absoluteCeilingRespected: requireTrue(input.absoluteCeilingRespected, `${expectedType} absoluteCeilingRespected`), + protectedReviewerAllocationRespected: requireTrue(input.protectedReviewerAllocationRespected, `${expectedType} protectedReviewerAllocationRespected`), + passClaimed: requireFalse(input.passClaimed, `${expectedType} passClaimed`), + }; +} + +function validateOnboardingFailure(value) { + const input = requireObject(value, "onboarding-remediation"); + if (!Number.isInteger(input.rank) || input.rank < 1 || input.rank > 5) { + throw new Error("onboarding-remediation rank must be from 1 to 5."); + } + if (!["fixed", "explicit-remediation"].includes(input.remediationStatus)) { + throw new Error("onboarding-remediation status is unsupported."); + } + return { + rank: input.rank, + code: requireSafeId(input.code, "onboarding-remediation code"), + remediationStatus: input.remediationStatus, + }; +} + +function validateContribution(value) { + const input = requireObject(value, "external-contribution"); + if (!["external-contribution", "substantive-issue"].includes(input.kind)) { + throw new Error("external-contribution kind is unsupported."); + } + return { + kind: input.kind, + externallyAuthored: requireTrue(input.externallyAuthored, "external-contribution externallyAuthored"), + }; +} + +function validateCaseStudy(value) { + const input = requireObject(value, "public-case-study"); + return { + id: requireSafeId(input.id, "public-case-study id"), + published: requireTrue(input.published, "public-case-study published"), + receiptExcerptIncluded: requireTrue(input.receiptExcerptIncluded, "public-case-study receiptExcerptIncluded"), + limitationsIncluded: requireTrue(input.limitationsIncluded, "public-case-study limitationsIncluded"), + usageTruthIncluded: requireTrue(input.usageTruthIncluded, "public-case-study usageTruthIncluded"), + }; +} + +function validateCalibration(value) { + const input = requireObject(value, "estimate-calibration-report"); + const confidencePromoted = input.confidencePromoted === true; + const minimumSampleMet = input.minimumSampleMet === true; + const driftCurrent = input.driftCurrent === true; + if (confidencePromoted && (!minimumSampleMet || !driftCurrent)) { + throw new Error("estimate confidence cannot be promoted without minimum samples and current drift evidence."); + } + return { + intervalCoverageReported: requireTrue(input.intervalCoverageReported, "estimate-calibration intervalCoverageReported"), + errorReported: requireTrue(input.errorReported, "estimate-calibration errorReported"), + taskClassBreakdown: requireTrue(input.taskClassBreakdown, "estimate-calibration taskClassBreakdown"), + confidencePromoted, + minimumSampleMet, + driftCurrent, + }; +} + +function validateGuardrailAudit(value) { + const input = requireObject(value, "guardrail-audit-pass"); + if (input.unresolvedBypasses !== 0) { + throw new Error("guardrail-audit-pass requires zero unresolved bypasses."); + } + return { + completed: requireTrue(input.completed, "guardrail-audit completed"), + unresolvedBypasses: 0, + }; +} + +function normalizeObservationData(value) { + switch (value.type) { + case "repository-attempt": return { attempt: validateRepositoryAttempt(value.attempt) }; + case "golden-path-complete": return { completion: validateGoldenPath(value.completion) }; + case "bounded-external-task": return { task: validateBoundedExternalTask(value.task) }; + case "repeat-user": return { repeatUse: validateRepeatUse(value.repeatUse) }; + case "native-goal-comparison": return { comparison: validateComparison(value.comparison) }; + case "measurable-benefit": return { benefit: validateBenefit(value.benefit) }; + case "recovery": return { recovery: validateRecovery(value.recovery) }; + case "operational-budget-exhaustion": return { pause: validatePause(value.pause, value.type) }; + case "controlled-host-limit": return { pause: validatePause(value.pause, value.type) }; + case "onboarding-remediation": return { failure: validateOnboardingFailure(value.failure) }; + case "external-contribution": return { contribution: validateContribution(value.contribution) }; + case "public-case-study": return { caseStudy: validateCaseStudy(value.caseStudy) }; + case "estimate-calibration-report": return { calibration: validateCalibration(value.calibration) }; + case "guardrail-audit-pass": return { audit: validateGuardrailAudit(value.audit) }; + default: throw new Error(`Unsupported pilot observation type: ${value.type || "missing"}.`); + } +} + +function validatePilotObservation(value) { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error("pilot observation must be a JSON object."); + } + if (value.schemaVersion !== PILOT_OBSERVATION_SCHEMA_VERSION) { + throw new Error(`Unsupported pilot observation schema: ${value.schemaVersion || "missing"}.`); + } + if (!OBSERVATION_TYPES.has(value.type)) { + throw new Error(`Unsupported pilot observation type: ${value.type || "missing"}.`); + } + return { + schemaVersion: PILOT_OBSERVATION_SCHEMA_VERSION, + id: requireSafeId(value.observationId, "pilot observation observationId"), + type: value.type, + observedAt: normalizeObservedAt(value.observedAt), + data: normalizeObservationData(value), + }; +} + +function loadPilotRecord(repoRoot, pilotId) { + const id = validatePilotId(pilotId); + const recordPath = path.join(path.resolve(repoRoot), ".cewp", "pilots", id, "record.json"); + if (!fs.existsSync(recordPath)) throw new Error(`Pilot record not found: ${id}.`); + const record = JSON.parse(fs.readFileSync(recordPath, "utf8")); + if (record.schemaVersion !== PILOT_RECORD_SCHEMA_VERSION || record.pilotId !== id) { + throw new Error(`Pilot record ${id} is malformed or incompatible.`); + } + return { record, recordPath }; +} + +function observationEvidenceIdentity(observation) { + if (observation.type === "repository-attempt") return `repository-attempt:${observation.data.attempt.id}`; + if (observation.type === "native-goal-comparison") return `native-goal-comparison:${observation.data.comparison.id}`; + if (observation.type === "recovery") return `recovery:${observation.data.recovery.id}`; + if (observation.type === "public-case-study") return `public-case-study:${observation.data.caseStudy.id}`; + if (observation.type === "onboarding-remediation") return `onboarding-remediation:${observation.data.failure.code}`; + return `${observation.type}:${observation.id}`; +} + +function existingObservationIdentities(repoRoot) { + const pilotsRoot = path.join(path.resolve(repoRoot), ".cewp", "pilots"); + const identities = new Set(); + if (!fs.existsSync(pilotsRoot)) return identities; + for (const entry of fs.readdirSync(pilotsRoot, { withFileTypes: true })) { + if (!entry.isDirectory()) continue; + const recordPath = path.join(pilotsRoot, entry.name, "record.json"); + if (!fs.existsSync(recordPath)) continue; + const record = JSON.parse(fs.readFileSync(recordPath, "utf8")); + for (const observation of record.observations || []) { + identities.add(`observation:${observation.id}`); + identities.add(`evidence:${observationEvidenceIdentity(observation)}`); + } + } + return identities; +} + +function recordPilotObservation(options = {}) { + const repoRoot = path.resolve(options.repoRoot || process.cwd()); + const source = readRepoJson(repoRoot, options.fromFile, "pilot observation --from"); + const observation = validatePilotObservation(source.value); + const found = loadPilotRecord(repoRoot, options.pilotId); + const identities = existingObservationIdentities(repoRoot); + const duplicateIdentity = identities.has(`observation:${observation.id}`) + ? `observation:${observation.id}` + : identities.has(`evidence:${observationEvidenceIdentity(observation)}`) + ? observationEvidenceIdentity(observation) + : null; + if (duplicateIdentity) { + throw new Error(`Pilot evidence identity already exists: ${duplicateIdentity}.`); + } + const eligible = found.record.participant.classification === "independent-external"; + const stored = { + ...observation, + recordedAt: (options.now || new Date()).toISOString(), + source: { sha256: source.sha256, pathRecorded: false }, + qualification: { + eligible, + classification: eligible ? "independent-evidence" : "maintainer-dogfood", + reason: eligible + ? "validated structured observation from an independent-external pilot record" + : "maintainer dogfood never counts as independent Phase 13 evidence", + }, + }; + const record = { + ...found.record, + observations: [...(found.record.observations || []), stored], + updatedAt: stored.recordedAt, + }; + writeJsonAtomic(found.recordPath, record); + return { record, observation: stored }; +} + +module.exports = { + OBSERVATION_TYPES, + PILOT_OBSERVATION_SCHEMA_VERSION, + loadPilotRecord, + observationEvidenceIdentity, + recordPilotObservation, + validatePilotObservation, +}; diff --git a/src/pilot/status.js b/src/pilot/status.js index 35af0a8..a4d2d46 100644 --- a/src/pilot/status.js +++ b/src/pilot/status.js @@ -44,11 +44,15 @@ function loadPilotRecords(repoRoot) { function derivePilotStatus(repoRoot) { const records = loadPilotRecords(repoRoot); const externalRecords = records.filter((record) => record.participant.classification === "independent-external"); - const externalParticipantIds = new Set(externalRecords.map((record) => record.participant.id)); + const externalParticipantIds = new Set(externalRecords + .filter((record) => (record.observations || []).some((observation) => ( + observation.type === "golden-path-complete" && observation.qualification?.eligible === true + ))) + .map((record) => record.participant.id)); const evidenceByType = new Map(); for (const record of externalRecords) { for (const observation of record.observations || []) { - if (!observation || observation.qualifies !== true || typeof observation.type !== "string") continue; + if (!observation || observation.qualification?.eligible !== true || typeof observation.type !== "string") continue; if (!evidenceByType.has(observation.type)) evidenceByType.set(observation.type, []); evidenceByType.get(observation.type).push(`${record.pilotId}:${observation.id || observation.type}`); } diff --git a/tests/contracts/pilot-gates.js b/tests/contracts/pilot-gates.js new file mode 100644 index 0000000..62a4199 --- /dev/null +++ b/tests/contracts/pilot-gates.js @@ -0,0 +1,251 @@ +"use strict"; + +const path = require("node:path"); +const { assert } = require("../harness/lib/assertions"); +const { cleanupRepo, makeTempRepo, runNode, writeJson } = require("../harness/lib/temp-repo"); + +const cewpCli = path.join(__dirname, "..", "..", "bin", "cewp.js"); + +function createPilot(repoRoot, pilotId, participant, participantId) { + const result = runNode(cewpCli, [ + "pilot", "create", + "--pilot-id", pilotId, + "--participant", participant, + "--participant-id", participantId, + "--json", + ], repoRoot); + assert(result.status === 0, `pilot ${pilotId} is created: ${result.stderr}`); +} + +function repositoryAttempt(observationId, attemptId) { + return { + schemaVersion: "pilot-observation/v1", + observationId, + type: "repository-attempt", + observedAt: "2026-07-22T02:00:00.000Z", + attempt: { + id: attemptId, + language: "javascript", + sizeBucket: "small", + operatingSystem: "windows", + testStack: "node", + inputKind: "direct-goal", + riskLevel: "low", + mode: "supervised", + }, + }; +} + +function goldenPath(observationId) { + return { + schemaVersion: "pilot-observation/v1", + observationId, + type: "golden-path-complete", + observedAt: "2026-07-22T03:00:00.000Z", + completion: { + supervised: true, + completed: true, + participantConfirmed: true, + firstApprovalMinutes: 4, + }, + }; +} + +function boundedExternalTask(observationId) { + return { + schemaVersion: "pilot-observation/v1", + observationId, + type: "bounded-external-task", + observedAt: "2026-07-22T04:00:00.000Z", + task: { + bounded: true, + realRepository: true, + acceptanceDefinedBeforeExecution: true, + }, + }; +} + +function recordObservation(repoRoot, pilotId, relativePath) { + return runNode(cewpCli, [ + "pilot", "record", pilotId, + "--from", relativePath, + "--yes", + "--json", + ], repoRoot); +} + +function recordInline(repoRoot, pilotId, observation) { + const fileName = `${observation.observationId}.json`; + writeJson(path.join(repoRoot, fileName), observation); + const result = recordObservation(repoRoot, pilotId, fileName); + assert(result.status === 0, `${observation.type} is recorded: ${result.stderr}`); + return JSON.parse(result.stdout).data.observation; +} + +function evidence(observationId, type, field, value, minute = 5) { + return { + schemaVersion: "pilot-observation/v1", + observationId, + type, + observedAt: `2026-07-22T05:${String(minute).padStart(2, "0")}:00.000Z`, + [field]: value, + }; +} + +function runContract() { + const repoRoot = makeTempRepo("cewp-pilot-gates-"); + try { + createPilot(repoRoot, "external-1", "independent-external", "person-1"); + createPilot(repoRoot, "dogfood-1", "maintainer-dogfood", "maintainer-1"); + writeJson(path.join(repoRoot, "external-attempt.json"), repositoryAttempt("external-attempt-1", "repo-attempt-1")); + writeJson(path.join(repoRoot, "dogfood-attempt.json"), repositoryAttempt("dogfood-attempt-1", "repo-attempt-dogfood")); + + const external = recordObservation(repoRoot, "external-1", "external-attempt.json"); + assert(external.status === 0, `external observation is recorded: ${external.stderr}`); + const externalOutput = JSON.parse(external.stdout).data; + assert(externalOutput.observation.qualification.eligible === true, "valid independent observation is eligible"); + + const dogfood = recordObservation(repoRoot, "dogfood-1", "dogfood-attempt.json"); + assert(dogfood.status === 0, `dogfood observation is recorded: ${dogfood.stderr}`); + const dogfoodOutput = JSON.parse(dogfood.stdout).data; + assert(dogfoodOutput.observation.qualification.eligible === false, "maintainer observation remains ineligible"); + + const statusResult = runNode(cewpCli, ["pilot", "status", "--json"], repoRoot); + const status = JSON.parse(statusResult.stdout).data; + const repositoryGate = status.gates.find((gate) => gate.id === "independent-repository-attempts"); + const participantGate = status.gates.find((gate) => gate.id === "independent-external-participants"); + assert(repositoryGate.observed === 1, "only the independent repository attempt counts"); + assert(repositoryGate.qualifyingEvidence.length === 1 && repositoryGate.qualifyingEvidence[0].includes("external-1"), "qualifying evidence is reviewable"); + assert(participantGate.observed === 0, "an enrolled external participant does not count before golden-path completion"); + + createPilot(repoRoot, "external-2", "independent-external", "person-2"); + createPilot(repoRoot, "external-3", "independent-external", "person-3"); + for (const [pilotId, fileName] of [ + ["external-1", "golden-1.json"], + ["external-2", "golden-2.json"], + ["external-3", "golden-3.json"], + ]) { + writeJson(path.join(repoRoot, fileName), goldenPath(`golden-${pilotId}`)); + const recorded = recordObservation(repoRoot, pilotId, fileName); + assert(recorded.status === 0, `${pilotId} golden path is recorded: ${recorded.stderr}`); + } + const afterGolden = JSON.parse(runNode(cewpCli, ["pilot", "status", "--json"], repoRoot).stdout).data; + const completedParticipants = afterGolden.gates.find((gate) => gate.id === "independent-external-participants"); + assert(completedParticipants.observed === 3 && completedParticipants.status === "met", "three distinct completed external participants satisfy the deferred gate"); + assert(afterGolden.complete === false, "participant completion alone cannot complete Phase 13"); + + writeJson(path.join(repoRoot, "bounded-task.json"), boundedExternalTask("bounded-task-1")); + const bounded = recordObservation(repoRoot, "external-1", "bounded-task.json"); + assert(bounded.status === 0, `bounded external task is recorded: ${bounded.stderr}`); + const afterBounded = JSON.parse(runNode(cewpCli, ["pilot", "status", "--json"], repoRoot).stdout).data; + const boundedGate = afterBounded.gates.find((gate) => gate.id === "real-bounded-external-repository-task"); + assert(boundedGate.observed === 1 && boundedGate.status === "met", "one real bounded external task satisfies its distinct gate"); + + for (let index = 2; index <= 10; index += 1) { + recordInline(repoRoot, `external-${((index - 1) % 3) + 1}`, repositoryAttempt(`external-attempt-${index}`, `repo-attempt-${index}`)); + } + for (let index = 1; index <= 3; index += 1) { + recordInline(repoRoot, `external-${index}`, evidence(`repeat-${index}`, "repeat-user", "repeatUse", { + runOrdinal: 2, + withoutMaintainerAssistance: true, + }, 10 + index)); + recordInline(repoRoot, `external-${index}`, evidence(`comparison-${index}`, "native-goal-comparison", "comparison", { + id: `comparison-${index}`, + equivalentTaskShape: true, + setupOverheadMeasured: true, + nativeUsage: "unknown", + cewpUsage: "observed", + }, 20 + index)); + recordInline(repoRoot, `external-${index}`, evidence(`recovery-${index}`, "recovery", "recovery", { + id: `recovery-${index}`, + scenario: index % 2 === 0 ? "failure-retry" : "pause-revise-resume", + recoveredWithoutRestart: true, + priorEvidenceRetained: true, + }, 30 + index)); + recordInline(repoRoot, `external-${index}`, evidence(`case-study-${index}`, "public-case-study", "caseStudy", { + id: `case-study-${index}`, + published: true, + receiptExcerptIncluded: true, + limitationsIncluded: true, + usageTruthIncluded: true, + }, 40 + index)); + } + recordInline(repoRoot, "external-1", evidence("benefit-1", "measurable-benefit", "benefit", { + dimension: "evidence-quality", + measured: true, + setupOverheadConsidered: true, + }, 50)); + recordInline(repoRoot, "external-1", evidence("budget-pause-1", "operational-budget-exhaustion", "pause", { + state: "paused-budget-unverified", + absoluteCeilingRespected: true, + protectedReviewerAllocationRespected: true, + passClaimed: false, + }, 51)); + recordInline(repoRoot, "external-1", evidence("host-limit-1", "controlled-host-limit", "pause", { + state: "paused-host-limit", + absoluteCeilingRespected: true, + protectedReviewerAllocationRespected: true, + passClaimed: false, + }, 52)); + for (let rank = 1; rank <= 5; rank += 1) { + recordInline(repoRoot, "external-1", evidence(`onboarding-${rank}`, "onboarding-remediation", "failure", { + rank, + code: `onboarding-${rank}`, + remediationStatus: rank % 2 === 0 ? "explicit-remediation" : "fixed", + }, 52 + rank)); + } + recordInline(repoRoot, "external-1", evidence("contribution-1", "external-contribution", "contribution", { + kind: "substantive-issue", + externallyAuthored: true, + }, 58)); + recordInline(repoRoot, "external-1", evidence("calibration-1", "estimate-calibration-report", "calibration", { + intervalCoverageReported: true, + errorReported: true, + taskClassBreakdown: true, + confidencePromoted: false, + minimumSampleMet: false, + driftCurrent: false, + }, 59)); + recordInline(repoRoot, "external-1", evidence("guardrail-audit-1", "guardrail-audit-pass", "audit", { + completed: true, + unresolvedBypasses: 0, + }, 59)); + + const broadStatus = JSON.parse(runNode(cewpCli, ["pilot", "status", "--json"], repoRoot).stdout).data; + const expectedMet = [ + "independent-repository-attempts", + "repeat-users-without-maintainer-assistance", + "comparable-native-goal-runs", + "measurable-cewp-benefit", + "recovered-pause-or-failure-scenarios", + "operational-budget-exhaustion", + "controlled-host-limit", + "top-onboarding-failures-remediated", + "external-contribution-or-substantive-issue", + "public-case-studies", + "usage-estimate-calibration-reported", + "guardrail-audit-with-no-unresolved-bypass", + ]; + for (const gateId of expectedMet) { + const gate = broadStatus.gates.find((entry) => entry.id === gateId); + assert(gate.status === "met", `${gateId} is evaluated from qualifying structured evidence`); + } + assert(broadStatus.gates.find((gate) => gate.id === "full-reviewed-runs").status === "unmet", "reviewed-run gate remains open for receipt-linked evidence"); + assert(broadStatus.complete === false, "broad fixture evidence cannot bypass the reviewed-run gate"); + + writeJson(path.join(repoRoot, "duplicate-attempt.json"), repositoryAttempt("duplicate-attempt-observation", "repo-attempt-1")); + const duplicateAttempt = recordObservation(repoRoot, "external-2", "duplicate-attempt.json"); + assert(duplicateAttempt.status === 1 && duplicateAttempt.stderr.includes("evidence identity already exists"), "duplicate repository attempts fail closed across pilot records"); + } finally { + cleanupRepo(repoRoot); + } +} + +try { + runContract(); + console.log("[PASS] structured pilot observation gates"); +} catch (error) { + console.error("[FAIL] structured pilot observation gates"); + console.error(error && error.stack ? error.stack : error); + process.exitCode = 1; +} From e72e8f0d1dd4d982533c8ec683c62d76c5b4d3c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tuncay=20=C3=96lmez?= Date: Sat, 25 Jul 2026 09:00:00 +0300 Subject: [PATCH 31/40] Require verified pilot run receipts --- package.json | 5 +- src/pilot/evidence.js | 84 ++++++++++++++ src/pilot/observation.js | 26 ++++- tests/contracts/pilot-receipt-link.js | 160 ++++++++++++++++++++++++++ 4 files changed, 270 insertions(+), 5 deletions(-) create mode 100644 src/pilot/evidence.js create mode 100644 tests/contracts/pilot-receipt-link.js diff --git a/package.json b/package.json index 488662f..d93e820 100644 --- a/package.json +++ b/package.json @@ -54,7 +54,7 @@ "scripts": { "test": "npm run test:contracts && npm run test:phase12 && npm run test:phase13 && npm run smoke", "test:phase12": "npm run test:evidence-receipt && npm run test:event-run-verify && npm run test:operator-report && npm run test:run-comparison && npm run test:evidence-redaction", - "test:phase13": "npm run test:pilot-record && npm run test:pilot-gates", + "test:phase13": "npm run test:pilot-record && npm run test:pilot-gates && npm run test:pilot-receipt-link", "test:contracts": "npm run test:init-install && npm run test:adapter-profile && npm run test:doctor-json && npm run test:operator-json && npm run test:hook-output && npm run test:skill-format && npm run test:ownership-gates && npm run test:fixtures && npm run test:plugin-package && npm run test:supervised-intake && npm run test:supervised-proposal && npm run test:supervised-controls && npm run test:supervised-execution && npm run test:supervised-review && npm run test:supervised-failure && npm run test:supervised-linear-resume && npm run test:supervised-demo && npm run test:workflow-definition && npm run test:workflow-compiler && npm run test:workflow-proposal && npm run test:workflow-scheduler && npm run test:workflow-worker-matrix && npm run test:workflow-result && npm run test:workflow-failure-result && npm run test:workflow-state-machine && npm run test:workflow-interventions && npm run test:workflow-failure-matrix && npm run test:workflow-budget && npm run test:workflow-progress && npm run test:workflow-review && npm run test:workflow-checkpoint-review && npm run test:workflow-lifecycle && npm run test:workflow-revision && npm run test:workflow-migration && npm run test:workflow-templates && npm run test:workflow-docs && npm run test:workflow-release && npm run test:integration-capabilities && npm run test:integration-binding && npm run test:integration-effort-policy && npm run test:integration-hook-evidence && npm run test:integration-mcp && npm run test:integration-observation && npm run test:native-goal-events", "test:init-install": "node tests/contracts/init-install.js", "test:adapter-profile": "node tests/contracts/adapter-profile.js", @@ -107,6 +107,7 @@ "test:native-goal-events": "node tests/contracts/native-goal-events.js", "test:pilot-record": "node tests/contracts/pilot-record.js", "test:pilot-gates": "node tests/contracts/pilot-gates.js", + "test:pilot-receipt-link": "node tests/contracts/pilot-receipt-link.js", "demo:supervised": "node src/demo/supervised.js", "test:plugin-lifecycle": "node tests/capabilities/plugin-lifecycle.js", "test:clean-install": "node tests/capabilities/clean-install.js", @@ -117,7 +118,7 @@ "check:workflow-syntax": "node --check ./src/workflow/budget.js && node --check ./src/workflow/compiler.js && node --check ./src/workflow/migration.js && node --check ./src/workflow/progress.js && node --check ./src/workflow/proposal.js && node --check ./src/workflow/review.js && node --check ./src/workflow/revision.js && node --check ./src/workflow/templates.js && node --check ./tests/contracts/workflow-budget.js && node --check ./tests/contracts/workflow-checkpoint-review.js && node --check ./tests/contracts/workflow-compiler.js && node --check ./tests/contracts/workflow-docs.js && node --check ./tests/contracts/workflow-failure-matrix.js && node --check ./tests/contracts/workflow-failure-result.js && node --check ./tests/contracts/workflow-migration.js && node --check ./tests/contracts/workflow-progress.js && node --check ./tests/contracts/workflow-release.js && node --check ./tests/contracts/workflow-review.js && node --check ./tests/contracts/workflow-lifecycle.js && node --check ./tests/contracts/workflow-revision.js && node --check ./tests/contracts/workflow-templates.js && node --check ./tests/contracts/workflow-worker-matrix.js", "check:mcp": "node --check ./bin/cewp-mcp.js && node --check ./src/mcp/server.js && node --check ./src/mcp/tools.js && node --check ./tests/contracts/integration-mcp.js", "check:evidence": "node --check ./src/evidence/receipt.js && node --check ./src/evidence/events.js && node --check ./src/evidence/report.js && node --check ./src/evidence/compare.js && node --check ./src/evidence/redaction.js && node --check ./src/evidence/usage.js && node --check ./src/evidence/verify.js && node --check ./src/run/verify.js && node --check ./tests/contracts/evidence-receipt.js && node --check ./tests/contracts/event-run-verify.js && node --check ./tests/contracts/operator-report.js && node --check ./tests/contracts/run-comparison.js && node --check ./tests/contracts/evidence-redaction.js", - "check:pilot": "node --check ./src/pilot/record.js && node --check ./src/pilot/observation.js && node --check ./src/pilot/status.js && node --check ./src/pilot/cli.js && node --check ./tests/contracts/pilot-record.js && node --check ./tests/contracts/pilot-gates.js", + "check:pilot": "node --check ./src/pilot/record.js && node --check ./src/pilot/observation.js && node --check ./src/pilot/evidence.js && node --check ./src/pilot/status.js && node --check ./src/pilot/cli.js && node --check ./tests/contracts/pilot-record.js && node --check ./tests/contracts/pilot-gates.js && node --check ./tests/contracts/pilot-receipt-link.js", "check": "npm run check:syntax && npm run check:workflow-syntax && npm run check:mcp && npm run check:evidence && npm run check:pilot && npm test", "pack:dry-run": "npm pack --dry-run" }, diff --git a/src/pilot/evidence.js b/src/pilot/evidence.js new file mode 100644 index 0000000..86b7c05 --- /dev/null +++ b/src/pilot/evidence.js @@ -0,0 +1,84 @@ +"use strict"; + +const crypto = require("node:crypto"); +const fs = require("node:fs"); +const path = require("node:path"); +const { verifyWorkflowRun } = require("../evidence/verify"); +const { loadWorkflowRun } = require("../workflow/state"); + +function sha256(filePath) { + return `sha256:${crypto.createHash("sha256").update(fs.readFileSync(filePath)).digest("hex")}`; +} + +function inspectReviewedRunEvidence(repoRoot, workflowRunId) { + let found; + try { + found = loadWorkflowRun(repoRoot, workflowRunId); + } catch (error) { + return { + status: "excluded", + reason: `workflow run is unavailable: ${error.message}`, + workflowRunId, + verification: { schemaVersion: "run-verification/v1", status: "failed", issues: [{ code: "run-unavailable", message: error.message }] }, + receipt: null, + rawEvidenceCopied: false, + }; + } + const verification = verifyWorkflowRun(repoRoot, workflowRunId); + const receiptPath = path.join(found.runRoot, "evidence-receipt.json"); + let receipt = null; + if (fs.existsSync(receiptPath)) { + try { + const value = JSON.parse(fs.readFileSync(receiptPath, "utf8")); + receipt = { + schemaVersion: value.schemaVersion || null, + generatedAt: value.generatedAt || null, + completeness: value.completeness?.status || "unknown", + integrityClaim: value.integrity?.claim || null, + sha256: sha256(receiptPath), + }; + } catch (error) { + receipt = { schemaVersion: null, generatedAt: null, completeness: "malformed", integrityClaim: null, sha256: sha256(receiptPath) }; + } + } + const reviewsRoot = path.join(found.runRoot, "reviews"); + const reviews = fs.existsSync(reviewsRoot) + ? fs.readdirSync(reviewsRoot, { withFileTypes: true }) + .filter((entry) => entry.isFile() && entry.name.endsWith(".json")) + .map((entry) => { + try { + return JSON.parse(fs.readFileSync(path.join(reviewsRoot, entry.name), "utf8")); + } catch (error) { + return null; + } + }) + .filter(Boolean) + : []; + const independentPass = reviews.some((review) => review.independent === true && review.decision === "PASS"); + let reason = null; + if (found.run.status !== "finalized") reason = `workflow run is not finalized (status ${found.run.status})`; + else if (!receipt) reason = "workflow run has no persisted evidence receipt"; + else if (receipt.schemaVersion !== "evidence-receipt/v1" || receipt.completeness !== "complete") reason = "workflow receipt is incomplete or incompatible"; + else if (verification.status !== "passed") reason = "workflow run verification failed"; + else if (found.run.reviewer?.decision !== "PASS" || !independentPass) reason = "workflow run lacks an independent reviewer PASS"; + return { + status: reason ? "excluded" : "qualified", + reason, + workflowRunId, + verification: { + schemaVersion: verification.schemaVersion, + status: verification.status, + issues: verification.issues.map((issue) => ({ code: issue.code, message: issue.message })), + }, + receipt, + reviewer: { + decision: found.run.reviewer?.decision || null, + independentPass, + }, + rawEvidenceCopied: false, + }; +} + +module.exports = { + inspectReviewedRunEvidence, +}; diff --git a/src/pilot/observation.js b/src/pilot/observation.js index e5aa52b..42c4cb1 100644 --- a/src/pilot/observation.js +++ b/src/pilot/observation.js @@ -5,6 +5,7 @@ const path = require("node:path"); const { readRepoJson } = require("../workflow/source"); const { writeJsonAtomic } = require("../workflow/state"); const { PILOT_RECORD_SCHEMA_VERSION, validatePilotId } = require("./record"); +const { inspectReviewedRunEvidence } = require("./evidence"); const PILOT_OBSERVATION_SCHEMA_VERSION = "pilot-observation/v1"; const SAFE_ID = /^[a-z0-9][a-z0-9-]{0,63}$/; @@ -23,6 +24,7 @@ const OBSERVATION_TYPES = new Set([ "public-case-study", "estimate-calibration-report", "guardrail-audit-pass", + "full-reviewed-run", ]); const ATTEMPT_ENUMS = Object.freeze({ sizeBucket: new Set(["small", "medium", "large", "unknown"]), @@ -255,6 +257,13 @@ function validateGuardrailAudit(value) { }; } +function validateReviewedRun(value) { + const input = requireObject(value, "full-reviewed-run"); + return { + workflowRunId: requireSafeId(input.workflowRunId, "full-reviewed-run workflowRunId"), + }; +} + function normalizeObservationData(value) { switch (value.type) { case "repository-attempt": return { attempt: validateRepositoryAttempt(value.attempt) }; @@ -271,6 +280,7 @@ function normalizeObservationData(value) { case "public-case-study": return { caseStudy: validateCaseStudy(value.caseStudy) }; case "estimate-calibration-report": return { calibration: validateCalibration(value.calibration) }; case "guardrail-audit-pass": return { audit: validateGuardrailAudit(value.audit) }; + case "full-reviewed-run": return { run: validateReviewedRun(value.run) }; default: throw new Error(`Unsupported pilot observation type: ${value.type || "missing"}.`); } } @@ -311,6 +321,7 @@ function observationEvidenceIdentity(observation) { if (observation.type === "recovery") return `recovery:${observation.data.recovery.id}`; if (observation.type === "public-case-study") return `public-case-study:${observation.data.caseStudy.id}`; if (observation.type === "onboarding-remediation") return `onboarding-remediation:${observation.data.failure.code}`; + if (observation.type === "full-reviewed-run") return `full-reviewed-run:${observation.data.run.workflowRunId}`; return `${observation.type}:${observation.id}`; } @@ -345,17 +356,26 @@ function recordPilotObservation(options = {}) { if (duplicateIdentity) { throw new Error(`Pilot evidence identity already exists: ${duplicateIdentity}.`); } - const eligible = found.record.participant.classification === "independent-external"; + const independentParticipant = found.record.participant.classification === "independent-external"; + const evidence = observation.type === "full-reviewed-run" + ? inspectReviewedRunEvidence(repoRoot, observation.data.run.workflowRunId) + : null; + const eligible = independentParticipant && (!evidence || evidence.status === "qualified"); const stored = { ...observation, recordedAt: (options.now || new Date()).toISOString(), source: { sha256: source.sha256, pathRecorded: false }, + evidence, qualification: { eligible, - classification: eligible ? "independent-evidence" : "maintainer-dogfood", + classification: eligible + ? "independent-evidence" + : independentParticipant ? "excluded-evidence" : "maintainer-dogfood", reason: eligible ? "validated structured observation from an independent-external pilot record" - : "maintainer dogfood never counts as independent Phase 13 evidence", + : independentParticipant + ? evidence.reason + : "maintainer dogfood never counts as independent Phase 13 evidence", }, }; const record = { diff --git a/tests/contracts/pilot-receipt-link.js b/tests/contracts/pilot-receipt-link.js new file mode 100644 index 0000000..f548eb9 --- /dev/null +++ b/tests/contracts/pilot-receipt-link.js @@ -0,0 +1,160 @@ +"use strict"; + +const fs = require("node:fs"); +const path = require("node:path"); +const { assert } = require("../harness/lib/assertions"); +const { cleanupRepo, makeTempRepo, runNode, writeFile, writeJson } = require("../harness/lib/temp-repo"); +const { validDefinition } = require("./workflow-definition"); +const { successfulResult } = require("./workflow-result"); +const { approveWorkflow } = require("./workflow-scheduler"); +const { + finalizeWorkflowRun, + loadWorkflowRun, + recordWorkflowResult, + recordWorkflowReview, + startWorkflowTask, +} = require("../../src/workflow/state"); +const { writeEvidenceReceipt } = require("../../src/evidence/receipt"); + +const cewpCli = path.join(__dirname, "..", "..", "bin", "cewp.js"); + +function createPilot(repoRoot, pilotId) { + const result = runNode(cewpCli, [ + "pilot", "create", + "--pilot-id", pilotId, + "--participant", "independent-external", + "--participant-id", `person-${pilotId.slice(-1)}`, + "--json", + ], repoRoot); + assert(result.status === 0, `pilot ${pilotId} is created: ${result.stderr}`); +} + +function completeRun(repoRoot, index) { + const definition = validDefinition(); + definition.workflowId = `pilot-reviewed-${index}`; + definition.tasks = [definition.tasks[0]]; + definition.budget.maxTargetedVerificationRuns = 6; + definition.budget.maxFullVerificationRuns = 1; + const approved = approveWorkflow(repoRoot, definition); + const approvedAt = Date.parse(approved.createdAt); + let found = loadWorkflowRun(repoRoot, approved.runId); + const started = startWorkflowTask(found, "implement-example", { + now: new Date(approvedAt + 60_000), + }); + const paths = { + baseline: `evidence/pilot-${index}-baseline.json`, + targeted: `evidence/pilot-${index}-targeted.json`, + artifact: `evidence/pilot-${index}-change.diff`, + review: `evidence/pilot-${index}-review.md`, + }; + writeFile(path.join(repoRoot, paths.baseline), "{\"status\":\"passed\"}\n"); + writeFile(path.join(repoRoot, paths.targeted), "{\"status\":\"passed\"}\n"); + writeFile(path.join(repoRoot, paths.artifact), "diff --git a/src/example.js b/src/example.js\n"); + writeFile(path.join(repoRoot, paths.review), "# Independent review\n\nPASS\n"); + const result = successfulResult(approved, started.checkpoint, [{ + command: "node --test tests/example.test.js", + status: "passed", + evidencePath: paths.targeted, + }]); + result.verification.baseline.evidence[0].evidencePath = paths.baseline; + result.artifacts[0].path = paths.artifact; + recordWorkflowResult(loadWorkflowRun(repoRoot, approved.runId), "implement-example", result); + const run = loadWorkflowRun(repoRoot, approved.runId).run; + recordWorkflowReview(loadWorkflowRun(repoRoot, approved.runId), { + schemaVersion: "review-result/v1", + reviewId: `${run.runId}-final-review`, + runId: run.runId, + workflowDigest: run.workflow.digest, + scope: { kind: "workflow", taskId: null, checkpointId: null }, + completedAt: new Date(approvedAt + 120_000).toISOString(), + independent: true, + decision: "PASS", + summary: "Independent pilot review passed.", + findings: [], + evidence: [{ kind: "review-report", path: paths.review }], + usage: { + managedOperations: { label: "observed", value: 1, source: "codex-exec-jsonl" }, + capturedOutputBytes: { label: "observed", value: 64, source: "cewp-bounded-output" }, + managedTokens: { label: "unknown", value: null, reason: "fixture omits token totals" }, + hostInternal: { label: "unknown", value: null, reason: "host usage unavailable" }, + }, + }); + finalizeWorkflowRun(loadWorkflowRun(repoRoot, approved.runId), { + now: new Date(approvedAt + 180_000), + }); + const finalized = loadWorkflowRun(repoRoot, approved.runId); + writeEvidenceReceipt(finalized, { generatedAt: new Date(approvedAt + 240_000).toISOString() }); + return { found: loadWorkflowRun(repoRoot, approved.runId), paths }; +} + +function reviewedObservation(observationId, workflowRunId) { + return { + schemaVersion: "pilot-observation/v1", + observationId, + type: "full-reviewed-run", + observedAt: "2026-07-22T07:00:00.000Z", + run: { workflowRunId }, + }; +} + +function recordObservation(repoRoot, pilotId, observation) { + const fileName = `${observation.observationId}.json`; + writeJson(path.join(repoRoot, fileName), observation); + return runNode(cewpCli, ["pilot", "record", pilotId, "--from", fileName, "--yes", "--json"], repoRoot); +} + +function runContract() { + const repoRoot = makeTempRepo("cewp-pilot-receipt-link-"); + try { + createPilot(repoRoot, "external-1"); + const complete = completeRun(repoRoot, 1); + const linked = recordObservation(repoRoot, "external-1", reviewedObservation("reviewed-run-1", complete.found.run.runId)); + assert(linked.status === 0, `complete reviewed run is linked: ${linked.stderr}`); + const linkedObservation = JSON.parse(linked.stdout).data.observation; + assert(linkedObservation.qualification.eligible === true, "complete receipt-linked run qualifies"); + assert(linkedObservation.evidence.receipt.schemaVersion === "evidence-receipt/v1", "link retains receipt contract identity"); + assert(/^sha256:[a-f0-9]{64}$/.test(linkedObservation.evidence.receipt.sha256), "link retains receipt digest without copying raw evidence"); + assert(linkedObservation.evidence.rawEvidenceCopied === false, "pilot link never copies raw workflow evidence"); + + const partialDefinition = validDefinition(); + partialDefinition.workflowId = "pilot-partial"; + const partial = approveWorkflow(repoRoot, partialDefinition); + const partialLink = recordObservation(repoRoot, "external-1", reviewedObservation("reviewed-run-partial", partial.runId)); + assert(partialLink.status === 0, `partial run remains recordable: ${partialLink.stderr}`); + const partialObservation = JSON.parse(partialLink.stdout).data.observation; + assert(partialObservation.qualification.eligible === false, "partial run cannot qualify the reviewed-run gate"); + assert(partialObservation.qualification.reason.includes("not finalized"), "partial run exclusion is actionable"); + + const status = JSON.parse(runNode(cewpCli, ["pilot", "status", "--json"], repoRoot).stdout).data; + const reviewedGate = status.gates.find((gate) => gate.id === "full-reviewed-runs"); + assert(reviewedGate.observed === 1 && reviewedGate.remaining === 4, "only the complete reviewed run counts"); + + const tampered = completeRun(repoRoot, 2); + fs.appendFileSync(path.join(repoRoot, tampered.paths.targeted), "tampered\n"); + const tamperedLink = recordObservation(repoRoot, "external-1", reviewedObservation("reviewed-run-tampered", tampered.found.run.runId)); + assert(tamperedLink.status === 0, "tampered run remains inspectable as excluded evidence"); + const tamperedObservation = JSON.parse(tamperedLink.stdout).data.observation; + assert(tamperedObservation.qualification.eligible === false, "receipt integrity failure cannot qualify"); + assert(tamperedObservation.evidence.verification.status === "failed", "integrity failure is retained in linked evidence"); + + for (let index = 3; index <= 6; index += 1) { + const completed = completeRun(repoRoot, index); + const recorded = recordObservation(repoRoot, "external-1", reviewedObservation(`reviewed-run-${index}`, completed.found.run.runId)); + assert(recorded.status === 0 && JSON.parse(recorded.stdout).data.observation.qualification.eligible === true, `reviewed run ${index} qualifies`); + } + const completedStatus = JSON.parse(runNode(cewpCli, ["pilot", "status", "--json"], repoRoot).stdout).data; + const completedGate = completedStatus.gates.find((gate) => gate.id === "full-reviewed-runs"); + assert(completedGate.observed === 5 && completedGate.status === "met", "five distinct verified reviewed runs satisfy the roadmap threshold"); + } finally { + cleanupRepo(repoRoot); + } +} + +try { + runContract(); + console.log("[PASS] pilot reviewed-run links require complete verified receipts"); +} catch (error) { + console.error("[FAIL] pilot reviewed-run receipt links"); + console.error(error && error.stack ? error.stack : error); + process.exitCode = 1; +} From 522a6ba35e16aca5143434cc37660d893d38d934 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tuncay=20=C3=96lmez?= Date: Sat, 25 Jul 2026 09:30:00 +0300 Subject: [PATCH 32/40] Add redacted pilot evidence exports --- package.json | 5 +- src/cli/parse.js | 5 ++ src/cli/usage.js | 1 + src/pilot/cli.js | 12 +++ src/pilot/export.js | 145 ++++++++++++++++++++++++++++++++ tests/contracts/pilot-export.js | 81 ++++++++++++++++++ 6 files changed, 247 insertions(+), 2 deletions(-) create mode 100644 src/pilot/export.js create mode 100644 tests/contracts/pilot-export.js diff --git a/package.json b/package.json index d93e820..6694fe0 100644 --- a/package.json +++ b/package.json @@ -54,7 +54,7 @@ "scripts": { "test": "npm run test:contracts && npm run test:phase12 && npm run test:phase13 && npm run smoke", "test:phase12": "npm run test:evidence-receipt && npm run test:event-run-verify && npm run test:operator-report && npm run test:run-comparison && npm run test:evidence-redaction", - "test:phase13": "npm run test:pilot-record && npm run test:pilot-gates && npm run test:pilot-receipt-link", + "test:phase13": "npm run test:pilot-record && npm run test:pilot-gates && npm run test:pilot-receipt-link && npm run test:pilot-export", "test:contracts": "npm run test:init-install && npm run test:adapter-profile && npm run test:doctor-json && npm run test:operator-json && npm run test:hook-output && npm run test:skill-format && npm run test:ownership-gates && npm run test:fixtures && npm run test:plugin-package && npm run test:supervised-intake && npm run test:supervised-proposal && npm run test:supervised-controls && npm run test:supervised-execution && npm run test:supervised-review && npm run test:supervised-failure && npm run test:supervised-linear-resume && npm run test:supervised-demo && npm run test:workflow-definition && npm run test:workflow-compiler && npm run test:workflow-proposal && npm run test:workflow-scheduler && npm run test:workflow-worker-matrix && npm run test:workflow-result && npm run test:workflow-failure-result && npm run test:workflow-state-machine && npm run test:workflow-interventions && npm run test:workflow-failure-matrix && npm run test:workflow-budget && npm run test:workflow-progress && npm run test:workflow-review && npm run test:workflow-checkpoint-review && npm run test:workflow-lifecycle && npm run test:workflow-revision && npm run test:workflow-migration && npm run test:workflow-templates && npm run test:workflow-docs && npm run test:workflow-release && npm run test:integration-capabilities && npm run test:integration-binding && npm run test:integration-effort-policy && npm run test:integration-hook-evidence && npm run test:integration-mcp && npm run test:integration-observation && npm run test:native-goal-events", "test:init-install": "node tests/contracts/init-install.js", "test:adapter-profile": "node tests/contracts/adapter-profile.js", @@ -108,6 +108,7 @@ "test:pilot-record": "node tests/contracts/pilot-record.js", "test:pilot-gates": "node tests/contracts/pilot-gates.js", "test:pilot-receipt-link": "node tests/contracts/pilot-receipt-link.js", + "test:pilot-export": "node tests/contracts/pilot-export.js", "demo:supervised": "node src/demo/supervised.js", "test:plugin-lifecycle": "node tests/capabilities/plugin-lifecycle.js", "test:clean-install": "node tests/capabilities/clean-install.js", @@ -118,7 +119,7 @@ "check:workflow-syntax": "node --check ./src/workflow/budget.js && node --check ./src/workflow/compiler.js && node --check ./src/workflow/migration.js && node --check ./src/workflow/progress.js && node --check ./src/workflow/proposal.js && node --check ./src/workflow/review.js && node --check ./src/workflow/revision.js && node --check ./src/workflow/templates.js && node --check ./tests/contracts/workflow-budget.js && node --check ./tests/contracts/workflow-checkpoint-review.js && node --check ./tests/contracts/workflow-compiler.js && node --check ./tests/contracts/workflow-docs.js && node --check ./tests/contracts/workflow-failure-matrix.js && node --check ./tests/contracts/workflow-failure-result.js && node --check ./tests/contracts/workflow-migration.js && node --check ./tests/contracts/workflow-progress.js && node --check ./tests/contracts/workflow-release.js && node --check ./tests/contracts/workflow-review.js && node --check ./tests/contracts/workflow-lifecycle.js && node --check ./tests/contracts/workflow-revision.js && node --check ./tests/contracts/workflow-templates.js && node --check ./tests/contracts/workflow-worker-matrix.js", "check:mcp": "node --check ./bin/cewp-mcp.js && node --check ./src/mcp/server.js && node --check ./src/mcp/tools.js && node --check ./tests/contracts/integration-mcp.js", "check:evidence": "node --check ./src/evidence/receipt.js && node --check ./src/evidence/events.js && node --check ./src/evidence/report.js && node --check ./src/evidence/compare.js && node --check ./src/evidence/redaction.js && node --check ./src/evidence/usage.js && node --check ./src/evidence/verify.js && node --check ./src/run/verify.js && node --check ./tests/contracts/evidence-receipt.js && node --check ./tests/contracts/event-run-verify.js && node --check ./tests/contracts/operator-report.js && node --check ./tests/contracts/run-comparison.js && node --check ./tests/contracts/evidence-redaction.js", - "check:pilot": "node --check ./src/pilot/record.js && node --check ./src/pilot/observation.js && node --check ./src/pilot/evidence.js && node --check ./src/pilot/status.js && node --check ./src/pilot/cli.js && node --check ./tests/contracts/pilot-record.js && node --check ./tests/contracts/pilot-gates.js && node --check ./tests/contracts/pilot-receipt-link.js", + "check:pilot": "node --check ./src/pilot/record.js && node --check ./src/pilot/observation.js && node --check ./src/pilot/evidence.js && node --check ./src/pilot/status.js && node --check ./src/pilot/export.js && node --check ./src/pilot/cli.js && node --check ./tests/contracts/pilot-record.js && node --check ./tests/contracts/pilot-gates.js && node --check ./tests/contracts/pilot-receipt-link.js && node --check ./tests/contracts/pilot-export.js", "check": "npm run check:syntax && npm run check:workflow-syntax && npm run check:mcp && npm run check:evidence && npm run check:pilot && npm test", "pack:dry-run": "npm pack --dry-run" }, diff --git a/src/cli/parse.js b/src/cli/parse.js index be9599e..5678ca1 100644 --- a/src/cli/parse.js +++ b/src/cli/parse.js @@ -111,6 +111,11 @@ function parseArgs(argv) { continue; } + if (args.command === "pilot" && args.subcommand === "export" && index === 2 && !arg.startsWith("--")) { + args.pilotId = arg; + continue; + } + if (args.command === "pilot" && arg === "--pilot-id") { const value = argv[index + 1]; if (!value || value.startsWith("--")) throw new Error("--pilot-id requires an id."); diff --git a/src/cli/usage.js b/src/cli/usage.js index e4efe1c..16c6838 100644 --- a/src/cli/usage.js +++ b/src/cli/usage.js @@ -10,6 +10,7 @@ Usage: cewp pilot create --pilot-id --participant --participant-id [--json] cewp pilot record --from --yes [--json] cewp pilot status [--json] + cewp pilot export [] [--json] cewp workflow validate [--json] cewp workflow compile (--goal | --from ) [--source-kind ] [--json] cewp workflow template [--json] diff --git a/src/pilot/cli.js b/src/pilot/cli.js index baa1a61..97ad647 100644 --- a/src/pilot/cli.js +++ b/src/pilot/cli.js @@ -3,6 +3,7 @@ const { createPilotRecord } = require("./record"); const { derivePilotStatus } = require("./status"); const { recordPilotObservation } = require("./observation"); +const { exportPilotEvidence } = require("./export"); function outputJson(command, data) { console.log(JSON.stringify({ @@ -15,6 +16,17 @@ function outputJson(command, data) { } function runPilot(options = {}) { + if (options.subcommand === "export") { + const result = exportPilotEvidence(process.cwd(), options.pilotId); + if (options.json) outputJson("pilot.export", result); + else { + console.log("CEWP redacted pilot export written"); + console.log(`Scope: ${result.export.scope.pilotId || "all"}`); + console.log(`JSON: ${result.paths.json}`); + console.log(`Markdown: ${result.paths.markdown}`); + } + return; + } if (options.subcommand === "record") { if (!options.yes) throw new Error("pilot record requires --yes confirmation."); const result = recordPilotObservation({ diff --git a/src/pilot/export.js b/src/pilot/export.js new file mode 100644 index 0000000..b3749eb --- /dev/null +++ b/src/pilot/export.js @@ -0,0 +1,145 @@ +"use strict"; + +const fs = require("node:fs"); +const path = require("node:path"); +const { redactEvidenceValue, REDACTION_SCHEMA_VERSION } = require("../evidence/redaction"); +const { ensureDir } = require("../lib/fs"); +const { normalizeSlashPath } = require("../lib/paths"); +const { writeJsonAtomic } = require("../workflow/state"); +const { validatePilotId } = require("./record"); +const { derivePilotStatus, loadPilotRecords } = require("./status"); + +const PILOT_EXPORT_SCHEMA_VERSION = "pilot-export/v1"; + +function isInside(parentPath, childPath) { + const relative = path.relative(path.resolve(parentPath), path.resolve(childPath)); + return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative)); +} + +function prepareContainedExportRoot(repoRoot, exportRoot) { + const resolvedRepo = path.resolve(repoRoot); + const resolvedExport = path.resolve(exportRoot); + if (!isInside(resolvedRepo, resolvedExport)) throw new Error("Pilot export root must stay inside the repository."); + const relativeParts = path.relative(resolvedRepo, resolvedExport).split(path.sep).filter(Boolean); + let cursor = resolvedRepo; + for (const part of relativeParts) { + cursor = path.join(cursor, part); + if (!fs.existsSync(cursor)) break; + if (fs.lstatSync(cursor).isSymbolicLink()) { + throw new Error(`Pilot export refuses a symbolic link in its output path: ${normalizeSlashPath(path.relative(resolvedRepo, cursor))}.`); + } + } + ensureDir(resolvedExport); + const realRepo = fs.realpathSync(resolvedRepo); + const realExport = fs.realpathSync(resolvedExport); + if (!isInside(realRepo, realExport)) throw new Error("Pilot export root resolves outside the repository."); +} + +function writeTextAtomic(filePath, value) { + ensureDir(path.dirname(filePath)); + const temporaryPath = `${filePath}.${process.pid}.tmp`; + fs.writeFileSync(temporaryPath, value, { flag: "wx" }); + try { + fs.renameSync(temporaryPath, filePath); + } catch (error) { + fs.rmSync(temporaryPath, { force: true }); + throw error; + } +} + +function renderPilotExportMarkdown(pilotExport) { + const lines = [ + "# CEWP Pilot Export", + "", + `Generated: ${pilotExport.generatedAt}`, + `Scope: ${pilotExport.scope.pilotId || "all local pilot records"}`, + `Phase 13 complete: ${pilotExport.status.complete ? "yes" : "no"}`, + "", + "## Privacy", + "", + `Redaction policy: ${pilotExport.redaction.schemaVersion}`, + `Replacements: ${pilotExport.redaction.replacements}`, + `Classes: ${pilotExport.redaction.classes.join(", ") || "none"}`, + "Raw prompts: excluded", + "Raw logs: excluded", + "Source code: excluded", + "", + "## Gates", + "", + "| Gate | Observed | Required | Status |", + "| --- | ---: | ---: | --- |", + ...pilotExport.status.gates.map((gate) => `| ${gate.id} | ${gate.observed} | ${gate.threshold} | ${gate.status} |`), + "", + "## Records", + "", + ]; + for (const record of pilotExport.records) { + lines.push(`### ${record.pilotId}`); + lines.push(""); + lines.push(`Participant classification: ${record.participant.classification}`); + lines.push(`Observation types: ${(record.observations || []).map((entry) => entry.type).join(", ") || "none"}`); + lines.push(""); + } + return `${lines.join("\n").trimEnd()}\n`; +} + +function buildPilotExport(repoRoot, pilotId, options = {}) { + const selectedId = pilotId ? validatePilotId(pilotId) : null; + const records = loadPilotRecords(repoRoot); + const selected = selectedId ? records.filter((record) => record.pilotId === selectedId) : records; + if (selectedId && selected.length === 0) throw new Error(`Pilot record not found: ${selectedId}.`); + const base = { + schemaVersion: PILOT_EXPORT_SCHEMA_VERSION, + generatedAt: (options.now || new Date()).toISOString(), + scope: { pilotId: selectedId, recordCount: selected.length }, + records: selected, + status: derivePilotStatus(repoRoot), + privacy: { + localCanonicalRootIncluded: false, + rawPromptsIncluded: false, + rawLogsIncluded: false, + sourceCodeIncluded: false, + authenticationMaterialIncluded: false, + patternRedactionIsProofOfNoSecrets: false, + }, + }; + const redacted = redactEvidenceValue(base); + return { + ...redacted.value, + redaction: { + schemaVersion: REDACTION_SCHEMA_VERSION, + applied: true, + replacements: redacted.replacements, + classes: redacted.classes, + canonicalRecordsModified: false, + }, + }; +} + +function exportPilotEvidence(repoRoot, pilotId, options = {}) { + const pilotExport = buildPilotExport(repoRoot, pilotId, options); + const exportId = pilotExport.scope.pilotId || "phase-13"; + const exportRoot = path.join(path.resolve(repoRoot), ".cewp", "pilot-exports", exportId); + const absolutePaths = { + json: path.join(exportRoot, "pilot-export.json"), + markdown: path.join(exportRoot, "pilot-export.md"), + }; + prepareContainedExportRoot(repoRoot, exportRoot); + writeJsonAtomic(absolutePaths.json, pilotExport); + writeTextAtomic(absolutePaths.markdown, renderPilotExportMarkdown(pilotExport)); + return { + export: pilotExport, + paths: Object.fromEntries(Object.entries(absolutePaths).map(([name, filePath]) => [ + name, + normalizeSlashPath(path.relative(path.resolve(repoRoot), filePath)), + ])), + }; +} + +module.exports = { + PILOT_EXPORT_SCHEMA_VERSION, + buildPilotExport, + exportPilotEvidence, + prepareContainedExportRoot, + renderPilotExportMarkdown, +}; diff --git a/tests/contracts/pilot-export.js b/tests/contracts/pilot-export.js new file mode 100644 index 0000000..6ed85b2 --- /dev/null +++ b/tests/contracts/pilot-export.js @@ -0,0 +1,81 @@ +"use strict"; + +const fs = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); +const { assert } = require("../harness/lib/assertions"); +const { cleanupRepo, makeTempRepo, readJson, runNode, writeJson } = require("../harness/lib/temp-repo"); + +const cewpCli = path.join(__dirname, "..", "..", "bin", "cewp.js"); +const SECRET = "sk-abcdefghijklmnopqrstuvwxyz123456"; + +function runContract() { + const repoRoot = makeTempRepo("cewp-pilot-export-"); + const outsideRoot = fs.mkdtempSync(path.join(os.tmpdir(), "cewp-pilot-export-outside-")); + try { + const created = runNode(cewpCli, [ + "pilot", "create", + "--pilot-id", "dogfood-1", + "--participant", "maintainer-dogfood", + "--participant-id", "maintainer-1", + "--json", + ], repoRoot); + assert(created.status === 0, `pilot fixture is created: ${created.stderr}`); + const canonicalPath = path.join(repoRoot, ".cewp", "pilots", "dogfood-1", "record.json"); + const canonical = readJson(canonicalPath); + canonical.adversarialFixture = { + apiKey: SECRET, + repositoryPath: "C:\\Users\\pilot\\private-repo", + note: ` Bearer ${SECRET}`, + }; + writeJson(canonicalPath, canonical); + const canonicalBefore = fs.readFileSync(canonicalPath, "utf8"); + + const exported = runNode(cewpCli, ["pilot", "export", "dogfood-1", "--json"], repoRoot); + assert(exported.status === 0, `pilot export succeeds: ${exported.stderr}`); + const output = JSON.parse(exported.stdout); + assert(output.command === "pilot.export", "pilot export has a stable command envelope"); + assert(output.data.export.schemaVersion === "pilot-export/v1", "pilot export is versioned"); + assert(output.data.export.records.length === 1 && output.data.export.records[0].pilotId === "dogfood-1", "selected export contains only the requested pilot record"); + assert(output.data.export.redaction.applied === true && output.data.export.redaction.replacements >= 3, "export discloses applied redaction and replacement count"); + assert(output.data.export.redaction.canonicalRecordsModified === false, "redaction never claims to modify canonical records"); + assert(output.data.export.privacy.rawPromptsIncluded === false && output.data.export.privacy.rawLogsIncluded === false, "raw prompts and logs are excluded"); + + const jsonPath = path.join(repoRoot, output.data.paths.json); + const markdownPath = path.join(repoRoot, output.data.paths.markdown); + assert(!path.isAbsolute(output.data.paths.json) && !output.data.paths.json.includes(".."), "export returns repository-relative contained paths"); + assert(fs.existsSync(jsonPath) && fs.existsSync(markdownPath), "separate JSON and Markdown exports are written"); + const exportedContents = `${fs.readFileSync(jsonPath, "utf8")}\n${fs.readFileSync(markdownPath, "utf8")}`; + for (const sensitive of [SECRET, "C:\\Users\\pilot\\private-repo", "