diff --git a/.gitignore b/.gitignore index 9d7148c..b99ec4e 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ research/ /review-board/ /security-gate/ +/compound-engineering/ /.review-board-*/ /.security-gate-*/ diff --git a/workflows/README.md b/workflows/README.md index 469d3e4..254db2a 100644 --- a/workflows/README.md +++ b/workflows/README.md @@ -33,7 +33,7 @@ From an Atomic chat session: Reporting workflows write their final Markdown report to disk and return compact metadata instead of returning the full report inline. -Reporting workflows save final reports under project-root output folders. Compound Engineering visible Markdown artifacts are collision-safe and use `-2`, `-3`, ... suffixes when a same-day slug already exists: +Reporting workflows save final reports under project-root output folders. Compound Engineering visible Markdown artifacts are collision-safe and use `-2`, `-3`, ... suffixes when a same-day slug already exists. `./compound-engineering/` is generated and gitignored, matching the generated `./review-board/` and `./security-gate/` report folders: ```text ./compound-engineering/YYYY-MM-DD-.md diff --git a/workflows/compound-engineering/README.md b/workflows/compound-engineering/README.md index 78d2b03..7462524 100644 --- a/workflows/compound-engineering/README.md +++ b/workflows/compound-engineering/README.md @@ -1,22 +1,81 @@ # compound-engineering -`compound-engineering` is a safe, artifact-backed Atomic workflow inspired by EveryInc's MIT-licensed [Compound Engineering Plugin](https://github.com/EveryInc/compound-engineering-plugin). It adapts the process vocabulary of brainstorm → plan → work → review → compound learning into an Atomic workflow graph. +`compound-engineering` is a safe, artifact-backed Atomic workflow inspired by EveryInc's MIT-licensed [Compound Engineering Plugin](https://github.com/EveryInc/compound-engineering-plugin). It adapts brainstorm → plan → work → review → compound learning into an Atomic workflow graph. -Iteration 3 is still safe by default, but explicit implementation runners are active after approval: it can classify intake, scout repo memory, write brainstorm/plan/spec/review/learning artifacts, ask for human approval, and then either stop at handoff metadata or launch the selected child workflow. File-only review artifacts are read from disk before evidence parsing, Ralph receives the effective Git worktree root, and validated implementation can capture learning according to `learning_mode`. +Iteration 4 remains safe by default while supporting explicit child runners after approval. The workflow classifies the request, scouts lightweight repo memory, writes file-backed brainstorm/plan/spec/review/learning artifacts, requires human approval, then either stops at handoff metadata or launches the selected declared child workflow. + +## Core patterns + +- **Classify and act:** `mode=auto` routes vague/product prompts to brainstorm, short concrete work prompts such as `Fix auth bug`, `Add API tests`, or `Fix CLI config` to work, plan/spec paths to work, review targets to review, and learning requests to compound-only. +- **Human-gated implementation:** no implementation before approval; no implementation runner starts before the approval prompt returns Approve. +- **Bounded loop:** `max_loops` is an integer bound for approval revisions and child runner loop metadata (`goal` maps it to `max_turns`). +- **Artifact handoffs:** specs, plans, final reports, learning docs, manifests, and child review reports stay file-backed; compact pointers must be read from disk before they count. +- **Adversarial verification:** parent completion depends on declared child outputs plus review artifacts, not writer prose or supplemental parent self-review. +- **Declared child contracts:** `goal` evidence is normalized only from declared Goal outputs; `ralph` evidence is normalized only from declared Ralph outputs. Unknown child keys are retained only as diagnostics (`unknown_child_output_keys`). +- **Fail-closed evidence extraction:** completion requires either a valid native builtin review-round JSON artifact or typed `review_evidence` in `review_report_path`, `review_report`, or another declared review artifact fallback (`ledger_path`, `implementation_notes_path`). Native review-round artifacts are parsed before wrapper `review_evidence` fallback only when the artifact has the native shape (`turn` or `iteration`, plus its own `reviews` property). Wrapper metadata such as `{ "iteration": 1, "review_evidence": { ... } }` remains wrapper evidence. Generic `result` prose is diagnostic only. Legacy `compound_engineering_evidence` and its entire subtree are ignored. Prose-only, malformed, missing, blocked, conflicted, stale, failed-validation, malformed stop-flag, reviewer-error, unknown-severity, or P0/P1 evidence cannot complete. +- **Strict numeric evidence:** validation `exit_code` values and `severity_counts.p0/p1/p2/p3` must be non-negative integers (and `exit_code` must be exactly `0` to pass). Fractional, non-finite, negative, string-coerced, malformed, or non-zero values fail closed; passing summaries do not override a bad `exit_code`. When no `exit_code` is present, explicit summary-only zero forms (`0 failures`, `zero failures`, `no failures`, `0 errors`, `zero errors`, `no errors`) can satisfy validation if there is no contradictory failure/error evidence. +- **Guard exits:** true precondition/child-exit guards use `ctx.exit` when available. Child-exit receipts include explicit `gate_child_run_completion` metadata with `state: "child_exited"`, `exit_status`, and `exit_reason`. Atomic child `cancelled` and `skipped` statuses are preserved in `ctx.exit.status` while the schema-compatible domain output status is `stopped`; blocked or unknown child exits use `ctx.exit.status="blocked"` and domain status `blocked`. Normal artifact-producing domain outcomes still return statuses such as `handoff_ready`, `needs_human`, `rejected`, `stopped`, and `complete`. ## Safe posture -- no implementation before approval: the workflow always requires a human approval gate before emitting implementation handoff metadata. -- `runner=auto` resolves to `handoff-only` in iteration 3, so the default remains non-mutating. +- `runner=auto` resolves to `handoff-only`, so the default remains non-mutating. - `runner=handoff-only` never launches a child workflow and omits command metadata unless there is a concrete command string. -- explicit `runner=goal` or `runner=ralph` runs after the human approval gate via Atomic built-ins. -- `goal` receives `objective`, `max_turns`, and `base_branch`. -- `ralph` receives `prompt`, `max_loops`, `base_branch`, strict boolean `create_pr`, and the effective Git worktree root derived with `git rev-parse --show-toplevel` when `git_worktree_dir` is non-empty. +- explicit `runner=goal` or `runner=ralph` runs only after the human approval gate via Atomic built-ins. +- `goal` receives `objective`, `max_turns`, and `base_branch`; completion requires `approved === true` and `status === "complete"`. +- `ralph` receives `prompt`, `max_loops`, `base_branch`, strict boolean `create_pr`, and the effective Git worktree root derived with `git rev-parse --show-toplevel` when `git_worktree_dir` is non-empty; completion requires `approved === true`. - `create_pr` defaults to `false`; only strict boolean `true` can authorize Ralph PR creation. -- Review and scout stages are read-only by prompt contract; implementation completion is gated by the child workflow's structured `compound_engineering_evidence`, not by parent supplemental review or writer self-attestation. -- File-only review stages save Markdown artifacts; compact inline strings such as `Output saved to: ...` are not treated as review evidence. -- Child implementation evidence must be emitted in child outputs and/or a review artifact as a `compound_engineering_evidence` JSON block with all six sufficiency criteria, severity counts, and validation command summaries/exit codes. Missing, malformed, prose-only, blocked, conflicted, failed-validation, or P0/P1 evidence fails closed. -- After a validated implementation, `learning_mode=lightweight|full` writes one `docs/solutions/...` artifact, `off` writes nothing, and `ask` presents the existing lightweight/full/off selection. +- Review/scout stages are read-only by prompt contract. Implementation completion is gated by structured review evidence from declared child outputs/artifacts. +- Legacy or undeclared `compound_engineering_evidence` child output is ignored as completion evidence, including all nested descendants; sibling declared `review_evidence` can still complete. +- After validated implementation, `learning_mode=lightweight|full` writes one `docs/solutions/...` artifact, `off` writes nothing, and `ask` presents the lightweight/full/off selection. + +## Child review evidence artifact shape + +A child review report can include JSON like: + +```json +{ + "review_evidence": { + "independent": { "satisfied": true, "evidence": "Fresh internal review after implementation." }, + "acceptance_mapped": { "satisfied": true, "evidence": "Each approved acceptance criterion was checked." }, + "diff_aware": { "satisfied": true, "evidence": "Changed files and git diff were inspected." }, + "validation_backed": { + "satisfied": true, + "evidence": "Validation passed.", + "commands": [{ "command": "bun test", "exit_code": 0, "summary": "passed" }] + }, + "risk_aware": { "satisfied": true, "evidence": "Residual risks documented." }, + "fresh": { "satisfied": true, "evidence": "Collected after the final diff." }, + "severity_counts": { "p0": 0, "p1": 0, "p2": 0, "p3": 0 } + } +} +``` + +Every satisfied criterion needs explicit support; validation commands must have integer zero exit codes or explicit passing summaries. If an `exit_code` field is present it must be exactly integer `0`; summaries are used only when no exit code is present. Accepted zero-count summaries include `0 failures`, `zero failures`, `no failures`, `0 errors`, `zero errors`, and `no errors`; contradictions such as `0 failures, 1 error` fail closed. Severity counts must be integer counts, not rounded or coerced. Optional stop flags `blocked`, `conflicted`, and `validation_failed` must be booleans when present; `false` is valid, while strings/numbers/null fail closed. + +Wrappers may include harmless metadata next to `review_evidence`; for example, `{ "iteration": 1, "review_evidence": { ... } }` is treated as wrapper evidence rather than a malformed native round. + +Native Goal/Ralph review-round artifacts are also accepted before wrapper evidence fallback when they have this JSON shape: + +```json +{ + "iteration": 2, + "reviews": [ + { + "reviewer": "native-reviewer", + "artifact_path": "/path/to/review.json", + "decision": { + "overall_correctness": "patch is correct", + "stop_review_loop": true, + "findings": [] + } + } + ] +} +``` + +Native review-round classification requires `turn` or `iteration` plus the artifact's own `reviews` property. Native review-round completion requires a non-empty `reviews` array, no root-level/review-level/decision-level `reviewer_error`, `overall_correctness: "patch is correct"`, any present `stop_review_loop` set to `true`, and no P0/P1 findings. Findings must use known integer/string severities P0-P3; unknown or malformed severities fail closed. The evidence trace records `source_kind: "native_goal_review_round"` or `"native_ralph_review_round"`. + +Prose fallback is negation-aware for hard-stop diagnostics: phrases such as `not blocked`, `no blockers`, `no blocking issues`, `not unable to review`, `no conflicting evidence`, `no conflicts`, `not conflicted`, and `no evidence conflicts` do not set blocked/conflicted flags, while positive blocker/conflict prose still does. Prose-only child evidence still cannot complete implementation. ## Inputs @@ -24,8 +83,8 @@ Iteration 3 is still safe by default, but explicit implementation runners are ac | --- | --- | --- | | `prompt` | required | Idea, spec/plan path, work request, review target, or learning-capture request. | | `mode` | `auto` | `auto`, `brainstorm`, `plan`, `work`, `review`, or `compound-only`. | -| `runner` | `auto` | `auto`, `goal`, `ralph`, or `handoff-only`. Auto is handoff-only in iteration 3; explicit `goal`/`ralph` run after approval. | -| `max_loops` | `5` | Bound for approval revision loops and child runner loops (`goal` maps it to `max_turns`). | +| `runner` | `auto` | `auto`, `goal`, `ralph`, or `handoff-only`. Auto is handoff-only; explicit `goal`/`ralph` run after approval. | +| `max_loops` | `5` | Integer bound for approval revision loops and child runner loops. | | `base_branch` | `origin/main` | Base branch copied into implementation handoff metadata. | | `git_worktree_dir` | `""` | Optional reusable worktree directory bound with `.worktreeFromInputs`; Ralph receives `""` when empty or the Git top-level root derived from `ctx.cwd` when non-empty. | | `create_pr` | `false` | Strict `true` only; safe default is no PR creation. | @@ -38,7 +97,7 @@ Required outputs: `status`, `mode`, `runner`, `approved`, `artifact_dir`, `manif Optional outputs: `brainstorm_path`, `plan_path`, `spec_path`, `approved_spec_path`, `implementation`, `review_report_path`, and `learning_doc_path`. -Statuses are honest fixed strings: `complete`, `approved`, `handoff_ready`, `review_only`, `blocked`, `needs_human`, `rejected`, or `stopped`. +Statuses are fixed strings: `complete`, `approved`, `handoff_ready`, `review_only`, `blocked`, `needs_human`, `rejected`, or `stopped`. ## Artifacts @@ -61,12 +120,13 @@ docs/solutions/YYYY-MM-DD-.md compound-engineering/YYYY-MM-DD-.md ``` -Visible Markdown writes use exclusive create and preserve existing files with `-2`, `-3`, ... suffixes. The manifest records sanitized inputs, timestamps, final report path, and artifact paths. +Visible Markdown writes use exclusive create and preserve existing files with `-2`, `-3`, ... suffixes. The `compound-engineering/` final report directory is generated and gitignored; hidden `/.compound-engineering-*/` run directories remain generated and gitignored separately. The manifest records sanitized inputs, timestamps, final report path, and artifact paths. ## Examples ```text /workflow compound-engineering prompt="Improve onboarding activation" mode=auto +/workflow compound-engineering prompt="Fix auth bug" mode=auto /workflow compound-engineering prompt="specs/2026-06-05-rate-limit.md" mode=work runner=ralph create_pr=false /workflow compound-engineering prompt="main..feature/auth" mode=review runner=handoff-only /workflow compound-engineering prompt="Capture lessons from the last fix" mode=compound-only learning_mode=lightweight diff --git a/workflows/compound-engineering/helpers.test.ts b/workflows/compound-engineering/helpers.test.ts index 1af32dd..38932bf 100644 --- a/workflows/compound-engineering/helpers.test.ts +++ b/workflows/compound-engineering/helpers.test.ts @@ -1,8 +1,13 @@ -import { mkdir, mkdtemp, readFile, realpath, rm, writeFile } from "node:fs/promises"; +import { mkdir, mkdtemp, readFile, realpath, readdir, rm, writeFile } from "node:fs/promises"; import { readFileSync } from "node:fs"; import { join } from "node:path"; import { tmpdir } from "node:os"; import { describe, expect, mock, test } from "bun:test"; +import { + gateChildRunCompletion, + parseEvidenceFromText, + reviewArtifactToEvidenceText, +} from "./lib/review-evidence.ts"; import { buildChildHandoff, datedMarkdownPath, @@ -63,7 +68,7 @@ function structuredEvidenceReport(evidence: Record = structured "P1: no findings", "", "```json", - JSON.stringify({ compound_engineering_evidence: evidence }, null, 2), + JSON.stringify({ review_evidence: evidence }, null, 2), "```", ].join("\n"); } @@ -84,6 +89,24 @@ function sufficientReviewReport(): string { return structuredEvidenceReport(); } +function nativeReviewRoundReport(overrides: Record = {}): string { + return JSON.stringify({ + iteration: 2, + reviews: [ + { + reviewer: "native-reviewer", + artifact_path: "/tmp/native-review.json", + decision: { + overall_correctness: "patch is correct", + stop_review_loop: true, + findings: [], + }, + }, + ], + ...overrides, + }); +} + type MockSchema = Record; type MockTypeOptions = Record; @@ -94,10 +117,13 @@ function mockSchema(type: string, options: MockTypeOptions = {}): MockSchema { const Type = { String: (options?: MockTypeOptions): MockSchema => mockSchema("string", options), Number: (options?: MockTypeOptions): MockSchema => mockSchema("number", options), + Integer: (options?: MockTypeOptions): MockSchema => mockSchema("integer", options), Boolean: (options?: MockTypeOptions): MockSchema => mockSchema("boolean", options), + Null: (options?: MockTypeOptions): MockSchema => mockSchema("null", options), Literal: (value: unknown): MockSchema => ({ const: value }), Union: (variants: MockSchema[], options: MockTypeOptions = {}): MockSchema => ({ anyOf: variants, ...options }), Object: (properties: Record, options: MockTypeOptions = {}): MockSchema => ({ type: "object", properties, ...options }), + Array: (items: MockSchema, options: MockTypeOptions = {}): MockSchema => ({ type: "array", items, ...options }), Optional: (schema: MockSchema): MockSchema => ({ ...schema, optional: true }), }; @@ -165,6 +191,9 @@ describe("compound-engineering mode routing", () => { expect(resolveMode("main..feature/auth", "auto")).toBe("review"); expect(resolveMode("Capture lessons learned from the rate limit fix", "auto")).toBe("compound-only"); expect(resolveMode("Improve onboarding activation", "auto")).toBe("brainstorm"); + expect(resolveMode("Fix auth bug", "auto")).toBe("work"); + expect(resolveMode("Add API tests", "auto")).toBe("work"); + expect(resolveMode("Fix CLI config", "auto")).toBe("work"); expect(resolveMode("Implement the TypeScript CLI config parser and add tests", "auto")).toBe("work"); }); }); @@ -222,7 +251,8 @@ describe("compound-engineering runner and approval helpers", () => { base_branch: "origin/main", }); expect(goalHandoff.inputs.objective).toContain("Implement the approved Compound Engineering plan/spec at specs/approved.md. Original request: Implement the approved spec"); - expect(goalHandoff.inputs.objective).toContain("compound_engineering_evidence"); + expect(goalHandoff.inputs.objective).toContain("review_evidence"); + expect(goalHandoff.inputs.objective).not.toContain("compound_engineering_evidence"); expect(goalHandoff.command).toContain("/workflow goal"); expect(goalHandoff.command).toContain("max_turns=5"); @@ -243,7 +273,6 @@ describe("compound-engineering runner and approval helpers", () => { describe("compound-engineering child completion and review artifact gates", () => { test("child completion gate vetoes non-approval statuses and missing approval", async () => { - const { gateChildRunCompletion } = await workflowModulePromise; expect(gateChildRunCompletion({ approved: false }, "ralph")).toMatchObject({ state: "non_approved", parent_status: "needs_human" }); for (const status of ["needs_human", "rejected", "stopped", "active"] as const) { @@ -254,9 +283,162 @@ describe("compound-engineering child completion and review artifact gates", () = } expect(gateChildRunCompletion({ status: "complete" }, "goal")).toMatchObject({ state: "missing_approval", parent_status: "needs_human" }); expect(gateChildRunCompletion({ approved: true, status: "mystery" }, "goal")).toMatchObject({ state: "non_approved", parent_status: "needs_human" }); + expect(gateChildRunCompletion({ approved: true }, "goal")).toMatchObject({ state: "missing_approval", parent_status: "needs_human" }); + expect(gateChildRunCompletion({ approved: true }, "ralph")).toMatchObject({ state: "approved" }); expect(gateChildRunCompletion({ approved: true, status: "complete" }, "goal")).toMatchObject({ state: "approved" }); }); + test("empty prompt guard uses ctx.exit before creating artifacts", async () => { + const dir = await mkdtemp(join(tmpdir(), "compound-engineering-empty-exit-")); + const workflow = await workflowPromise; + try { + let exitPayload: Record | undefined; + const result = await workflow.run({ + cwd: dir, + inputs: { + prompt: " ", + mode: "auto", + runner: "auto", + max_loops: 2, + base_branch: "origin/main", + git_worktree_dir: "", + create_pr: false, + learning_mode: "off", + memory_scope: "none", + }, + ui: { select: async () => "Approve", input: async () => "" }, + task: async () => ({ text: "should not run" }), + workflow: async () => ({}), + exit: (payload: Record) => { + exitPayload = payload; + return (payload.outputs ?? {}) as Record; + }, + }); + + expect(exitPayload).toMatchObject({ status: "blocked", reason: "Blocked: prompt is required." }); + expect(result).toMatchObject({ status: "blocked", approved: false, artifact_dir: "", manifest_path: "" }); + expect(await readdir(dir)).toEqual([]); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + test("child exited guard preserves Atomic exit status while keeping domain status schema-compatible", async () => { + const cases: Array<{ name: string; childResult: Record; exitStatus: string; domainStatus: string; exitReason: string }> = [ + { name: "cancelled", childResult: { exited: true, status: "cancelled", reason: "user cancelled" }, exitStatus: "cancelled", domainStatus: "stopped", exitReason: "user cancelled" }, + { name: "skipped", childResult: { exited: true, exit: { status: "skipped", reason: "precondition skipped" } }, exitStatus: "skipped", domainStatus: "stopped", exitReason: "precondition skipped" }, + { name: "blocked", childResult: { exited: true, status: "blocked", reason: "child blocked" }, exitStatus: "blocked", domainStatus: "blocked", exitReason: "child blocked" }, + { name: "exit-reason-precedence", childResult: { exited: true, status: "blocked", exitReason: "top Atomic exitReason", reason: "alias reason", exit_reason: "snake alias reason", exit: { reason: "nested reason" } }, exitStatus: "blocked", domainStatus: "blocked", exitReason: "top Atomic exitReason" }, + { name: "reason-precedence", childResult: { exited: true, status: "blocked", reason: "top reason", exit_reason: "snake alias reason", exit: { reason: "nested reason" } }, exitStatus: "blocked", domainStatus: "blocked", exitReason: "top reason" }, + { name: "snake-precedence", childResult: { exited: true, status: "blocked", exit_reason: "snake alias reason", exit: { reason: "nested reason" } }, exitStatus: "blocked", domainStatus: "blocked", exitReason: "snake alias reason" }, + { name: "unknown", childResult: { exited: true, outputs: { approved: true, status: "complete", review_report: sufficientReviewReport() } }, exitStatus: "blocked", domainStatus: "blocked", exitReason: "goal child workflow exited before returning declared completion evidence." }, + ]; + + for (const testCase of cases) { + const dir = await mkdtemp(join(tmpdir(), `compound-engineering-child-exited-${testCase.name}-`)); + const workflow = await workflowPromise; + try { + let exitPayload: Record | undefined; + const result = await workflow.run({ + cwd: dir, + inputs: { + prompt: "Implement approved thing", + mode: "work", + runner: "goal", + max_loops: 2, + base_branch: "origin/main", + git_worktree_dir: "", + create_pr: false, + learning_mode: "off", + memory_scope: "none", + }, + ui: { select: async () => "Approve", input: async () => "" }, + task: async (_name: string, options: { output?: string }) => { + if (options.output) await writeMarkdown(options.output, "task output"); + return { text: "draft" }; + }, + workflow: async () => testCase.childResult, + exit: (payload: Record) => { + exitPayload = payload; + return (payload.outputs ?? {}) as Record; + }, + }); + + expect(exitPayload).toMatchObject({ status: testCase.exitStatus, reason: testCase.exitReason }); + expect(result.status).toBe(testCase.domainStatus); + expect((result.implementation as Record).child_exited).toBe(true); + expect((result.implementation as Record).gate_child_run_completion).toMatchObject({ + state: "child_exited", + parent_status: testCase.domainStatus, + exited: true, + exit_status: testCase.exitStatus, + reason: testCase.exitReason, + exit_reason: testCase.exitReason, + }); + } finally { + await rm(dir, { recursive: true, force: true }); + } + } + }); + + test("implementation receipt schema allows null leaves and array items", async () => { + const workflow = await workflowPromise; + const implementation = workflow.outputs.implementation; + const variants = implementation.anyOf as MockSchema[]; + const childReceipt = variants.find((variant) => ((variant.properties as Record).kind as MockSchema).const === "child_workflow_receipt") as MockSchema; + const outputs = ((childReceipt.properties as Record).outputs as MockSchema); + const leaf = outputs.additionalProperties as MockSchema; + const leafVariants = leaf.anyOf as MockSchema[]; + const arrayVariant = leafVariants.find((variant) => variant.type === "array") as MockSchema; + const arrayItemVariants = ((arrayVariant.items as MockSchema).anyOf as MockSchema[]); + + expect(leafVariants.some((variant) => variant.type === "null")).toBe(true); + expect(arrayItemVariants.some((variant) => variant.type === "null")).toBe(true); + }); + + test("implementation receipt preserves declared nulls and compacted array nulls", async () => { + const dir = await mkdtemp(join(tmpdir(), "compound-engineering-receipt-null-")); + const workflow = await workflowPromise; + try { + const result = await workflow.run({ + cwd: dir, + inputs: { + prompt: "Implement approved thing", + mode: "work", + runner: "goal", + max_loops: 2, + base_branch: "origin/main", + git_worktree_dir: "", + create_pr: false, + learning_mode: "off", + memory_scope: "none", + }, + ui: { select: async () => "Approve", input: async () => "" }, + task: async (_name: string, options: { output?: string }) => { + if (options.output) await writeMarkdown(options.output, "task output"); + return { text: "draft" }; + }, + workflow: async () => ({ + outputs: { + result: "implementation done", + approved: true, + status: "complete", + ledger_path: null, + remaining_work: [null, Number.POSITIVE_INFINITY, undefined, () => "ignored", Symbol("ignored"), "done"], + review_report: sufficientReviewReport(), + }, + }), + }); + + expect(result.status).toBe("complete"); + const outputs = ((result.implementation as Record).outputs as Record); + expect(outputs.ledger_path).toBeNull(); + expect(outputs.remaining_work).toEqual([null, null, null, null, null, "done"]); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + test("supplemental review cannot override child approved false", async () => { const dir = await mkdtemp(join(tmpdir(), "compound-engineering-child-veto-")); const workflow = await workflowPromise; @@ -303,7 +485,7 @@ describe("compound-engineering child completion and review artifact gates", () = const reviewPath = join(dir, "ralph-review.json"); await writeFile(reviewPath, JSON.stringify({ reviewer: "ralph", - compound_engineering_evidence: structuredEvidence(), + review_evidence: structuredEvidence(), }), "utf8"); const result = await workflow.run({ @@ -398,7 +580,7 @@ describe("compound-engineering child completion and review artifact gates", () = if (options.output) await writeMarkdown(options.output, sufficientReviewReport()); return { text: "draft" }; }, - workflow: async () => ({ outputs: { approved: true, review_report: `Latest review round artifact: ${missingReviewPath}` } }), + workflow: async () => ({ outputs: { approved: true, status: "complete", review_report: `Latest review round artifact: ${missingReviewPath}` } }), }); expect(supplementalTaskRan).toBe(false); @@ -437,7 +619,7 @@ describe("compound-engineering child completion and review artifact gates", () = if (options.output) await writeMarkdown(options.output, "task output"); return { text: "draft" }; }, - workflow: async () => ({ outputs: { approved: true, review_report: `Latest review round artifact: ${reviewPath}` } }), + workflow: async () => ({ outputs: { approved: true, status: "complete", review_report: `Latest review round artifact: ${reviewPath}` } }), }); expect(supplementalTaskRan).toBe(false); @@ -477,7 +659,7 @@ describe("compound-engineering child completion and review artifact gates", () = if (options.output) await writeMarkdown(options.output, "task output"); return { text: "draft" }; }, - workflow: async () => ({ outputs: { approved: true, review_report: `Latest review round artifact: ${reviewPath}` } }), + workflow: async () => ({ outputs: { approved: true, status: "complete", review_report: `Latest review round artifact: ${reviewPath}` } }), }); expect(supplementalTaskRan).toBe(false); @@ -523,7 +705,7 @@ describe("compound-engineering child completion and review artifact gates", () = if (options.output) await writeMarkdown(options.output, "task output"); return { text: "draft" }; }, - workflow: async () => ({ outputs: { approved: true, review_report: `Output saved to: ${reviewPath} (48.2 KB, 2847 lines). Read this file if needed.` } }), + workflow: async () => ({ outputs: { approved: true, status: "complete", review_report: `Output saved to: ${reviewPath} (48.2 KB, 2847 lines). Read this file if needed.` } }), }); expect(supplementalTaskRan).toBe(false); @@ -566,7 +748,7 @@ describe("compound-engineering child completion and review artifact gates", () = if (options.output) await writeMarkdown(options.output, sufficientReviewReport()); return { text: "draft" }; }, - workflow: async () => ({ outputs: { approved: true, review_report: `Saved output to: ${missingReviewPath}` } }), + workflow: async () => ({ outputs: { approved: true, status: "complete", review_report: `Saved output to: ${missingReviewPath}` } }), }); expect(supplementalTaskRan).toBe(false); @@ -662,6 +844,7 @@ describe("compound-engineering child completion and review artifact gates", () = outputs: { result: "implementation done", approved: true, + status: "complete", review_report: sufficientReviewReport(), review_report_path: undefined, ledger_path: ledgerPath, @@ -675,13 +858,14 @@ describe("compound-engineering child completion and review artifact gates", () = expect(result.status).toBe("complete"); const implementation = result.implementation as Record; const outputs = implementation.outputs as Record; - expect(outputs.ledger_path).toBe(ledgerPath); + expect(outputs).not.toHaveProperty("ledger_path"); expect(() => JSON.stringify(result)).not.toThrow(); expect(outputs).not.toHaveProperty("raw"); expect(outputs).not.toHaveProperty("plan"); expect(outputs).not.toHaveProperty("ledger"); expect(outputs).not.toHaveProperty("receipts"); - expect(outputs.omitted_child_output_keys).toEqual(expect.arrayContaining(["ledger", "plan", "receipts"])); + expect(outputs.unknown_child_output_keys).toEqual(expect.arrayContaining(["ledger", "ledger_path", "receipts", "status"])); + expect(outputs.omitted_inline_child_output_keys).toEqual(expect.arrayContaining(["plan"])); expect(JSON.stringify(result)).not.toContain(rawPlanSentinel); expect(JSON.stringify(result)).not.toContain(rawLedgerSentinel); expect(JSON.stringify(result)).not.toContain(rawReceiptSentinel); @@ -736,9 +920,10 @@ describe("compound-engineering child completion and review artifact gates", () = expect(() => JSON.stringify(result)).not.toThrow(); const outputs = ((result.implementation as Record).outputs as Record); expect(outputs.result).toBe("123"); - expect(outputs.artifact_dir).toBe("/tmp/child-artifacts"); - expect(outputs.validation_output).toMatchObject({ kept_path: "/tmp/validation.log", nested: { command: "bun test" }, self: "[Circular]" }); - expect(outputs.changed_files).toEqual(["src/index.ts", null, null, null, null, null, "7"]); + expect(outputs).not.toHaveProperty("artifact_dir"); + expect(outputs).not.toHaveProperty("validation_output"); + expect(outputs).not.toHaveProperty("changed_files"); + expect(outputs.unknown_child_output_keys).toEqual(expect.arrayContaining(["artifact_dir", "changed_files", "validation_output"])); } finally { await rm(dir, { recursive: true, force: true }); } @@ -776,6 +961,7 @@ describe("compound-engineering child completion and review artifact gates", () = outputs: { result: "implementation done", approved: true, + status: "complete", review_report: sufficientReviewReport(), validation_output: validationOutput, }, @@ -785,13 +971,8 @@ describe("compound-engineering child completion and review artifact gates", () = expect(result.status).toBe("complete"); expect(() => JSON.stringify(result)).not.toThrow(); const outputs = ((result.implementation as Record).outputs as Record); - const compactValidationOutput = outputs.validation_output as Record; - expect(compactValidationOutput.key_00).toBe("value_0"); - expect(compactValidationOutput.key_49).toBe("value_49"); - expect(compactValidationOutput).not.toHaveProperty("key_55"); - expect(compactValidationOutput.original_keys).toBe(60); - expect(compactValidationOutput.omitted_key_count).toBe(10); - expect(compactValidationOutput.omitted_keys).toEqual(expect.arrayContaining(["key_55"])); + expect(outputs).not.toHaveProperty("validation_output"); + expect(outputs.unknown_child_output_keys).toEqual(expect.arrayContaining(["status", "validation_output"])); expect(JSON.stringify(result)).not.toContain(omittedValueSentinel); const manifest = JSON.parse(await readFile(result.manifest_path as string, "utf8")); @@ -805,7 +986,6 @@ describe("compound-engineering child completion and review artifact gates", () = describe("compound-engineering review evidence parser and reducer", () => { test("severity parser ignores empty headings and counts real finding lines", async () => { - const { parseEvidenceFromText } = await workflowModulePromise; const evidence = parseEvidenceFromText([ "## P0", "P0: none", @@ -827,7 +1007,6 @@ describe("compound-engineering review evidence parser and reducer", () => { }); test("validation availability alone does not satisfy validation-backed evidence", async () => { - const { parseEvidenceFromText } = await workflowModulePromise; const evidence = parseEvidenceFromText([ "Independent separate reviewer mapped acceptance criteria against the approved spec.", "Inspected git diff and changed files.", @@ -845,7 +1024,6 @@ describe("compound-engineering review evidence parser and reducer", () => { }); test("bare validation execution wording does not satisfy validation-backed evidence", async () => { - const { parseEvidenceFromText } = await workflowModulePromise; const evidence = parseEvidenceFromText([ "Independent separate reviewer mapped acceptance criteria against the approved spec.", "Inspected git diff and changed files.", @@ -864,7 +1042,6 @@ describe("compound-engineering review evidence parser and reducer", () => { }); test("negated validation success wording fails closed before success matching", async () => { - const { parseEvidenceFromText } = await workflowModulePromise; const negatedSuccessLines = [ "Validation commands run: bun test not successful.", "Validation commands run: bun test not passed.", @@ -892,7 +1069,6 @@ describe("compound-engineering review evidence parser and reducer", () => { }); test("validation no-run wording fails closed instead of satisfying validation evidence", async () => { - const { parseEvidenceFromText } = await workflowModulePromise; const noRunLines = [ "Validation commands run: none.", "Validation commands run: no commands.", @@ -925,7 +1101,6 @@ describe("compound-engineering review evidence parser and reducer", () => { }); test("failed validation outcomes set validation_failed and cannot satisfy validation evidence", async () => { - const { parseEvidenceFromText } = await workflowModulePromise; const failureLines = [ "Validation commands run: bun test did not pass.", "Validation commands run: bun test reported 2 failures.", @@ -965,7 +1140,6 @@ describe("compound-engineering review evidence parser and reducer", () => { }); test("adjacent validation exit code and status segments fail closed", async () => { - const { parseEvidenceFromText } = await workflowModulePromise; const adjacentFailures = [ "Validation commands run: bun test. Exit code: 1.", "Validation commands run: bun test; status: 2.", @@ -991,7 +1165,6 @@ describe("compound-engineering review evidence parser and reducer", () => { }); test("contextual validation errors fail but expected negative-path errors can pass", async () => { - const { parseEvidenceFromText } = await workflowModulePromise; const contextualErrors = [ "Validation commands run: bun test reported 3 errors.", "Validation commands run: bun test had 3 errors.", @@ -1033,7 +1206,6 @@ describe("compound-engineering review evidence parser and reducer", () => { }); test("explicit pass and status-zero wording satisfies validation-backed evidence", async () => { - const { parseEvidenceFromText } = await workflowModulePromise; const successLines = [ "Validation commands run: bun test passed.", "Validation commands run: bun test succeeded.", @@ -1060,7 +1232,6 @@ describe("compound-engineering review evidence parser and reducer", () => { }); test("negated validation failure wording does not set validation_failed", async () => { - const { parseEvidenceFromText } = await workflowModulePromise; const noFailureLines = [ "Validation commands run: bun test passed; no validation commands failed.", "Validation commands run: bun test passed; No tests failed.", @@ -1094,7 +1265,6 @@ describe("compound-engineering review evidence parser and reducer", () => { }); test("Ralph JSON review artifacts expose findings and raw text as evidence", async () => { - const { parseEvidenceFromText, reviewArtifactToEvidenceText } = await workflowModulePromise; const evidenceText = reviewArtifactToEvidenceText(JSON.stringify({ reviewer: "reviewer-a", decision: { @@ -1118,7 +1288,6 @@ describe("compound-engineering review evidence parser and reducer", () => { }); test("acceptance evidence requires explicit mapping, checking, tracing, verification, or coverage", async () => { - const { parseEvidenceFromText } = await workflowModulePromise; const bareMentions = parseEvidenceFromText([ "Independent separate reviewer completed the review.", "The latest approved spec and criteria were mentioned.", @@ -1148,7 +1317,6 @@ describe("compound-engineering review evidence parser and reducer", () => { }); test("freshness requires review after the final or current diff/latest change", async () => { - const { parseEvidenceFromText } = await workflowModulePromise; expect(parseEvidenceFromText("Latest change is mentioned in the review summary.").fresh).toBeUndefined(); expect(parseEvidenceFromText("Reviewed after latest change.").fresh).toBe(true); @@ -1159,7 +1327,6 @@ describe("compound-engineering review evidence parser and reducer", () => { }); test("diff unavailable wording prevents diff-aware sufficiency", async () => { - const { parseEvidenceFromText } = await workflowModulePromise; const unavailableLines = [ "Unable to inspect git diff for this run.", "Could not inspect changed files.", @@ -1190,7 +1357,6 @@ describe("compound-engineering review evidence parser and reducer", () => { }); test("negated and skipped review phrases do not satisfy evidence criteria", async () => { - const { parseEvidenceFromText } = await workflowModulePromise; const evidence = parseEvidenceFromText([ "Review is not independent.", "No acceptance mapping was provided.", @@ -1215,7 +1381,6 @@ describe("compound-engineering review evidence parser and reducer", () => { }); test("negated criterion variants stay missing even with other affirmative evidence", async () => { - const { parseEvidenceFromText } = await workflowModulePromise; const positiveLines = { independent: "Independent separate reviewer completed the review.", acceptance_mapped: "Acceptance criteria were mapped to the approved spec.", @@ -1248,7 +1413,6 @@ describe("compound-engineering review evidence parser and reducer", () => { }); test("common negated acceptance and validation variants do not satisfy sufficiency", async () => { - const { parseEvidenceFromText } = await workflowModulePromise; const evidence = parseEvidenceFromText([ "Independent separate reviewer completed the review.", "No acceptance criteria were mapped.", @@ -1268,7 +1432,6 @@ describe("compound-engineering review evidence parser and reducer", () => { }); test("common negated independent, acceptance, diff, and test variants fail closed", async () => { - const { parseEvidenceFromText } = await workflowModulePromise; const evidence = parseEvidenceFromText([ "No separate reviewer completed this.", "Acceptance mapping was not provided.", @@ -1290,7 +1453,6 @@ describe("compound-engineering review evidence parser and reducer", () => { }); test("affirmative non-negated review evidence still satisfies all criteria", async () => { - const { parseEvidenceFromText } = await workflowModulePromise; const evidence = parseEvidenceFromText([ "Independent separate reviewer mapped acceptance criteria against the approved spec.", "Inspected git diff and changed files.", @@ -1327,6 +1489,21 @@ describe("compound-engineering review evidence parser and reducer", () => { expect(reduction.missing).toEqual([]); }); + test("fractional severity counts fail closed instead of being floored", () => { + const reduction = reduceReviewEvidence({ + independent: true, + acceptance_mapped: true, + diff_aware: true, + validation_backed: true, + risk_aware: true, + fresh: true, + severity_counts: { p0: 0, p1: 0, p2: 0.5, p3: 0 }, + }); + + expect(reduction.decision).toBe("needs_human"); + expect(reduction.reason).toContain("non-negative integers"); + }); + test("one missing non-fresh criterion asks for targeted review", () => { const reduction = reduceReviewEvidence({ independent: true, @@ -1361,6 +1538,37 @@ describe("compound-engineering review evidence parser and reducer", () => { }).decision).toBe("full_review"); }); + test("negated prose hard-stop phrases do not set blocked or conflicted flags", async () => { + const negatedBlocked = ["not blocked", "no blockers", "no blocking issues", "not unable to review"]; + const negatedConflicted = ["no conflicting evidence", "no conflicts", "not conflicted", "no evidence conflicts"]; + + for (const phrase of negatedBlocked) { + expect(parseEvidenceFromText(`Review complete and ${phrase}.`).blocked).toBeUndefined(); + } + for (const phrase of negatedConflicted) { + expect(parseEvidenceFromText(`Review complete with ${phrase}.`).conflicted).toBeUndefined(); + } + }); + + test("positive prose hard-stop phrases still set blocked or conflicted flags", async () => { + const blockedPhrases = ["blocked", "blocker remains", "blocking issue", "unable to review", "missing dependency"]; + const conflictedPhrases = ["conflicting evidence", "evidence conflict", "conflicted", "review conflicts with results"]; + + for (const phrase of blockedPhrases) { + const evidence = parseEvidenceFromText(`Review is ${phrase}.`); + expect(evidence.blocked).toBe(true); + expect(reduceReviewEvidence(evidence).decision).toBe("blocked"); + } + for (const phrase of conflictedPhrases) { + const evidence = parseEvidenceFromText(`Review has ${phrase}.`); + expect(evidence.conflicted).toBe(true); + expect(reduceReviewEvidence(evidence).decision).toBe("needs_human"); + } + + expect(parseEvidenceFromText("Not blocked, but a blocker remains.").blocked).toBe(true); + expect(parseEvidenceFromText("No conflicts, but evidence conflicts with validation.").conflicted).toBe(true); + }); + test("blocking severities, conflicts, validation failures, and blockers fail closed", () => { expect(reduceReviewEvidence({ severity_counts: { p1: 1 } }).decision).toBe("fixes_needed"); expect(reduceReviewEvidence({ conflicted: true }).decision).toBe("needs_human"); @@ -1478,8 +1686,11 @@ describe("compound-engineering artifact path helpers and discoverability", () => expect(workflowReadme()).toContain("EveryInc"); expect(workflowReadme()).toContain("no implementation before approval"); expect(workflowReadme()).toContain("explicit `runner=goal` or `runner=ralph`"); + expect(workflowReadme()).toContain("`compound-engineering/` final report directory is generated and gitignored"); expect(registryReadme()).toContain("compound-engineering"); + expect(registryReadme()).toContain("`./compound-engineering/` is generated and gitignored"); expect(rootReadme()).toContain("compound-engineering"); + expect(rootGitignore()).toContain("/compound-engineering/"); expect(rootGitignore()).toContain("/.compound-engineering-*/"); }); @@ -1520,6 +1731,7 @@ describe("compound-engineering artifact path helpers and discoverability", () => result: "implementation done", review_report: sufficientReviewReport(), approved: true, + status: "complete", }, }; }, @@ -1653,6 +1865,402 @@ describe("compound-engineering artifact path helpers and discoverability", () => } }); + test("native goal and ralph review-round artifacts complete without synthetic review_evidence", async () => { + for (const runner of ["goal", "ralph"] as const) { + const dir = await mkdtemp(join(tmpdir(), `compound-engineering-native-${runner}-`)); + const workflow = await workflowPromise; + try { + const result = await workflow.run({ + cwd: dir, + inputs: { + prompt: "Implement approved thing", + mode: "work", + runner, + max_loops: 2, + base_branch: "origin/main", + git_worktree_dir: "", + create_pr: false, + learning_mode: "off", + memory_scope: "none", + }, + ui: { select: async () => "Approve", input: async () => "" }, + task: async (_name: string, options: { output?: string }) => { + if (options.output) await writeMarkdown(options.output, "task output"); + return { text: "draft" }; + }, + workflow: async () => ({ outputs: { result: "implementation done", review_report: nativeReviewRoundReport(), approved: true, status: "complete" } }), + }); + + expect(result.status).toBe("complete"); + expect(result.approved).toBe(true); + expect(((result.implementation as Record).evidence as Record).structured_child).toMatchObject({ + contract: "native_builtin_review_round", + source_kind: runner === "goal" ? "native_goal_review_round" : "native_ralph_review_round", + loaded: true, + errors: [], + }); + } finally { + await rm(dir, { recursive: true, force: true }); + } + } + }); + + test("review_evidence wrappers with iteration metadata complete without native shadowing", async () => { + const dir = await mkdtemp(join(tmpdir(), "compound-engineering-wrapper-iteration-")); + const workflow = await workflowPromise; + try { + const report = JSON.stringify({ iteration: 1, review_evidence: structuredEvidence() }); + const result = await workflow.run({ + cwd: dir, + inputs: { + prompt: "Implement approved thing", + mode: "work", + runner: "goal", + max_loops: 2, + base_branch: "origin/main", + git_worktree_dir: "", + create_pr: false, + learning_mode: "off", + memory_scope: "none", + }, + ui: { select: async () => "Approve", input: async () => "" }, + task: async (_name: string, options: { output?: string }) => { + if (options.output) await writeMarkdown(options.output, "task output"); + return { text: "draft" }; + }, + workflow: async () => ({ outputs: { result: "implementation done", review_report: report, approved: true, status: "complete" } }), + }); + + expect(result.status).toBe("complete"); + expect(result.approved).toBe(true); + expect(((result.implementation as Record).evidence as Record).structured_child).toMatchObject({ + contract: "declared_child_review_artifact", + loaded: true, + errors: [], + }); + expect(JSON.stringify(result.implementation)).not.toContain("native_goal_review_round"); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + test("legacy compound_engineering_evidence subtree is ignored but sibling review_evidence can complete", async () => { + const cases: Array<{ name: string; report: string; expectedStatus: string; expectedMessage?: string }> = [ + { + name: "legacy-only", + report: JSON.stringify({ compound_engineering_evidence: structuredEvidence() }), + expectedStatus: "needs_human", + expectedMessage: "legacy compound_engineering_evidence was ignored", + }, + { + name: "nested-legacy", + report: JSON.stringify({ wrapper: { compound_engineering_evidence: { review_evidence: structuredEvidence() } } }), + expectedStatus: "needs_human", + expectedMessage: "legacy compound_engineering_evidence was ignored", + }, + { + name: "sibling-review-evidence", + report: JSON.stringify({ review_evidence: structuredEvidence(), compound_engineering_evidence: { review_evidence: structuredEvidence({ blocked: true }) } }), + expectedStatus: "complete", + }, + ]; + + for (const testCase of cases) { + const dir = await mkdtemp(join(tmpdir(), `compound-engineering-legacy-${testCase.name}-`)); + const workflow = await workflowPromise; + try { + const result = await workflow.run({ + cwd: dir, + inputs: { + prompt: "Implement approved thing", + mode: "work", + runner: "goal", + max_loops: 2, + base_branch: "origin/main", + git_worktree_dir: "", + create_pr: false, + learning_mode: "off", + memory_scope: "none", + }, + ui: { select: async () => "Approve", input: async () => "" }, + task: async (_name: string, options: { output?: string }) => { + if (options.output) await writeMarkdown(options.output, "task output"); + return { text: "draft" }; + }, + workflow: async () => ({ outputs: { result: "implementation done", review_report: testCase.report, approved: true, status: "complete" } }), + }); + + expect(result.status).toBe(testCase.expectedStatus); + expect(result.approved).toBe(testCase.expectedStatus === "complete"); + const structuredChild = ((result.implementation as Record).evidence as Record).structured_child as Record; + expect(structuredChild.legacy_compound_engineering_evidence_ignored).toBe(true); + if (testCase.expectedMessage) expect(result.message).toContain(testCase.expectedMessage); + if (testCase.expectedStatus === "complete") { + expect(((result.implementation as Record).gate_review_evidence as Record).decision).toBe("sufficient"); + } + } finally { + await rm(dir, { recursive: true, force: true }); + } + } + }); + + test("native review-round artifacts fail closed on blocking or malformed decisions", async () => { + const cases: Array<{ name: string; report: string; native: boolean; decision?: string; expectedMessage?: string }> = [ + { + name: "p0-finding", + report: nativeReviewRoundReport({ reviews: [{ reviewer: "native", decision: { overall_correctness: "patch is correct", findings: [{ title: "data loss", priority: 0 }] } }] }), + native: true, + decision: "fixes_needed", + }, + { + name: "p1-finding", + report: nativeReviewRoundReport({ reviews: [{ reviewer: "native", decision: { overall_correctness: "patch is correct", findings: [{ title: "security bug", severity: "P1" }] } }] }), + native: true, + decision: "fixes_needed", + }, + { + name: "root-reviewer-error", + report: nativeReviewRoundReport({ reviewer_error: "round failed" }), + native: true, + }, + { + name: "review-reviewer-error", + report: nativeReviewRoundReport({ reviews: [{ reviewer: "native", reviewer_error: "review failed", decision: { overall_correctness: "patch is correct", findings: [] } }] }), + native: true, + }, + { + name: "decision-reviewer-error", + report: nativeReviewRoundReport({ reviews: [{ reviewer: "native", decision: { overall_correctness: "patch is correct", reviewer_error: "tool failed", findings: [] } }] }), + native: true, + }, + { + name: "bad-correctness", + report: nativeReviewRoundReport({ reviews: [{ reviewer: "native", decision: { overall_correctness: "patch is incorrect", findings: [] } }] }), + native: true, + }, + { + name: "stop-loop-false", + report: nativeReviewRoundReport({ reviews: [{ reviewer: "native", decision: { overall_correctness: "patch is correct", stop_review_loop: false, findings: [] } }] }), + native: true, + }, + { + name: "malformed-decision", + report: nativeReviewRoundReport({ reviews: [{ reviewer: "native", decision: "approved" }] }), + native: true, + }, + { + name: "empty-reviews", + report: JSON.stringify({ iteration: 1, reviews: [] }), + native: true, + }, + { + name: "iteration-only-metadata", + report: JSON.stringify({ iteration: 1 }), + native: false, + expectedMessage: "missing review_evidence object", + }, + { + name: "unknown-severity", + report: nativeReviewRoundReport({ reviews: [{ reviewer: "native", decision: { overall_correctness: "patch is correct", findings: [{ title: "mystery", severity: "critical" }] } }] }), + native: true, + }, + ]; + + for (const testCase of cases) { + const dir = await mkdtemp(join(tmpdir(), `compound-engineering-native-${testCase.name}-`)); + const workflow = await workflowPromise; + try { + const result = await workflow.run({ + cwd: dir, + inputs: { + prompt: "Implement approved thing", + mode: "work", + runner: "goal", + max_loops: 2, + base_branch: "origin/main", + git_worktree_dir: "", + create_pr: false, + learning_mode: "off", + memory_scope: "none", + }, + ui: { select: async () => "Approve", input: async () => "" }, + task: async (_name: string, options: { output?: string }) => { + if (options.output) await writeMarkdown(options.output, "task output"); + return { text: "draft" }; + }, + workflow: async () => ({ outputs: { result: "implementation done", review_report: testCase.report, approved: true, status: "complete" } }), + }); + + expect(result.status).not.toBe("complete"); + expect(result.approved).toBe(false); + const structuredChild = ((result.implementation as Record).evidence as Record).structured_child as Record; + if (testCase.native) { + expect(structuredChild).toMatchObject({ source_kind: "native_goal_review_round" }); + } else { + expect(structuredChild).not.toHaveProperty("source_kind"); + } + if (testCase.decision) expect(((result.implementation as Record).gate_review_evidence as Record).decision).toBe(testCase.decision); + if (testCase.expectedMessage) expect(result.message).toContain(testCase.expectedMessage); + } finally { + await rm(dir, { recursive: true, force: true }); + } + } + }); + + test("structured validation summary-only zero failure and no-error phrases can satisfy", async () => { + const dir = await mkdtemp(join(tmpdir(), "compound-engineering-zero-summary-")); + const workflow = await workflowPromise; + try { + const summaries = ["0 failures", "zero failures", "no failures", "0 errors", "zero errors", "no errors"]; + const result = await workflow.run({ + cwd: dir, + inputs: { + prompt: "Implement approved thing", + mode: "work", + runner: "goal", + max_loops: 2, + base_branch: "origin/main", + git_worktree_dir: "", + create_pr: false, + learning_mode: "off", + memory_scope: "none", + }, + ui: { select: async () => "Approve", input: async () => "" }, + task: async (_name: string, options: { output?: string }) => { + if (options.output) await writeMarkdown(options.output, "task output"); + return { text: "draft" }; + }, + workflow: async () => ({ + outputs: { + result: "implementation done", + review_report: structuredEvidenceReport(structuredEvidence({ + validation_backed: { + satisfied: true, + evidence: "Validation command summaries reported zero failures/errors.", + commands: summaries.map((summary) => ({ command: "validation", summary })), + }, + })), + approved: true, + status: "complete", + }, + }), + }); + + expect(result.status).toBe("complete"); + expect(result.approved).toBe(true); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + test("structured validation summary contradictions fail closed", async () => { + const dir = await mkdtemp(join(tmpdir(), "compound-engineering-contradictory-summary-")); + const workflow = await workflowPromise; + try { + const result = await workflow.run({ + cwd: dir, + inputs: { + prompt: "Implement approved thing", + mode: "work", + runner: "goal", + max_loops: 2, + base_branch: "origin/main", + git_worktree_dir: "", + create_pr: false, + learning_mode: "off", + memory_scope: "none", + }, + ui: { select: async () => "Approve", input: async () => "" }, + task: async (_name: string, options: { output?: string }) => { + if (options.output) await writeMarkdown(options.output, "task output"); + return { text: "draft" }; + }, + workflow: async () => ({ + outputs: { + result: "implementation done", + review_report: structuredEvidenceReport(structuredEvidence({ + validation_backed: { + satisfied: true, + evidence: "Validation summary contradicted itself.", + commands: [{ command: "bun test", summary: "0 failures, 1 error" }], + }, + })), + approved: true, + status: "complete", + }, + }), + }); + + expect(result.status).toBe("needs_human"); + expect(result.approved).toBe(false); + expect(result.message).toContain("validation_backed.commands"); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + test("structured numeric evidence requires integer exit codes and severity counts", async () => { + const cases = [ + { + name: "fractional-exit-code", + evidence: structuredEvidence({ + validation_backed: { satisfied: true, evidence: "Validation passed according to summary.", commands: [{ command: "bun test", exit_code: 0.5, summary: "bun test passed" }] }, + }), + expectedMessage: "validation_backed.commands", + }, + { + name: "string-exit-code", + evidence: structuredEvidence({ + validation_backed: { satisfied: true, evidence: "Validation passed according to summary.", commands: [{ command: "bun test", exit_code: "0", summary: "bun test passed" }] }, + }), + expectedMessage: "validation_backed.commands", + }, + { + name: "fractional-severity-count", + evidence: structuredEvidence({ severity_counts: { p0: 0, p1: 0, p2: 0.5, p3: 0 } }), + expectedMessage: "severity_counts.p2", + }, + { + name: "negative-severity-count", + evidence: structuredEvidence({ severity_counts: { p0: 0, p1: 0, p2: -1, p3: 0 } }), + expectedMessage: "severity_counts.p2", + }, + ]; + + for (const testCase of cases) { + const dir = await mkdtemp(join(tmpdir(), `compound-engineering-${testCase.name}-`)); + const workflow = await workflowPromise; + try { + const result = await workflow.run({ + cwd: dir, + inputs: { + prompt: "Implement approved thing", + mode: "work", + runner: "goal", + max_loops: 2, + base_branch: "origin/main", + git_worktree_dir: "", + create_pr: false, + learning_mode: "off", + memory_scope: "none", + }, + ui: { select: async () => "Approve", input: async () => "" }, + task: async (_name: string, options: { output?: string }) => { + if (options.output) await writeMarkdown(options.output, "task output"); + return { text: "draft" }; + }, + workflow: async () => ({ outputs: { result: "implementation done", review_report: structuredEvidenceReport(testCase.evidence), approved: true, status: "complete" } }), + }); + + expect(result.status).toBe("needs_human"); + expect(result.approved).toBe(false); + expect(result.message).toContain(testCase.expectedMessage); + } finally { + await rm(dir, { recursive: true, force: true }); + } + } + }); + test("structured evidence with all criteria true returns complete", async () => { const dir = await mkdtemp(join(tmpdir(), "compound-engineering-structured-complete-")); const workflow = await workflowPromise; @@ -1677,7 +2285,7 @@ describe("compound-engineering artifact path helpers and discoverability", () => if (options.output) await writeMarkdown(options.output, "task output"); return { text: "draft" }; }, - workflow: async () => ({ outputs: { result: "implementation done", compound_engineering_evidence: structuredEvidence(), approved: true } }), + workflow: async () => ({ outputs: { result: "implementation done", review_report: structuredEvidenceReport(), approved: true, status: "complete" } }), }); expect(supplementalTaskRan).toBe(false); @@ -1714,7 +2322,7 @@ describe("compound-engineering artifact path helpers and discoverability", () => if (options.output) await writeMarkdown(options.output, "task output"); return { text: "draft" }; }, - workflow: async () => ({ outputs: { result: "implementation done with incomplete review evidence", compound_engineering_evidence: incompleteEvidence, approved: true } }), + workflow: async () => ({ outputs: { result: "implementation done with incomplete review evidence", review_report: structuredEvidenceReport(incompleteEvidence), approved: true, status: "complete" } }), }); expect(supplementalTaskRan).toBe(false); @@ -1750,7 +2358,7 @@ describe("compound-engineering artifact path helpers and discoverability", () => if (options.output) await writeMarkdown(options.output, "task output"); return { text: "draft" }; }, - workflow: async () => ({ outputs: { result: "implementation done", compound_engineering_evidence: structuredEvidence({ validation_failed: true }), approved: true } }), + workflow: async () => ({ outputs: { result: "implementation done", review_report: structuredEvidenceReport(structuredEvidence({ validation_failed: true })), approved: true, status: "complete" } }), }); expect(result.status).toBe("needs_human"); @@ -1784,7 +2392,7 @@ describe("compound-engineering artifact path helpers and discoverability", () => if (options.output) await writeMarkdown(options.output, "task output"); return { text: "draft" }; }, - workflow: async () => ({ outputs: { result: "implementation done", compound_engineering_evidence: structuredEvidence({ severity_counts: { p0: severity === "p0" ? 1 : 0, p1: severity === "p1" ? 1 : 0, p2: 0, p3: 0 } }), approved: true } }), + workflow: async () => ({ outputs: { result: "implementation done", review_report: structuredEvidenceReport(structuredEvidence({ severity_counts: { p0: severity === "p0" ? 1 : 0, p1: severity === "p1" ? 1 : 0, p2: 0, p3: 0 } })), approved: true, status: "complete" } }), }); expect(result.status).toBe("needs_human"); @@ -1796,6 +2404,51 @@ describe("compound-engineering artifact path helpers and discoverability", () => } }); + test("structured stop flags fail closed when malformed and accept false booleans", async () => { + const cases: Array<{ name: string; evidence: Record; expectedStatus: string; expectedMessage?: string }> = [ + { name: "blocked-string", evidence: structuredEvidence({ blocked: "false" }), expectedStatus: "needs_human", expectedMessage: "blocked must be boolean" }, + { name: "conflicted-number", evidence: structuredEvidence({ conflicted: 0 }), expectedStatus: "needs_human", expectedMessage: "conflicted must be boolean" }, + { name: "validation-failed-string", evidence: structuredEvidence({ validation_failed: "false" }), expectedStatus: "needs_human", expectedMessage: "validation_failed must be boolean" }, + { name: "false-booleans", evidence: structuredEvidence({ blocked: false, conflicted: false, validation_failed: false }), expectedStatus: "complete" }, + ]; + + for (const testCase of cases) { + const dir = await mkdtemp(join(tmpdir(), `compound-engineering-stop-flags-${testCase.name}-`)); + const workflow = await workflowPromise; + try { + const result = await workflow.run({ + cwd: dir, + inputs: { + prompt: "Implement approved thing", + mode: "work", + runner: "goal", + max_loops: 2, + base_branch: "origin/main", + git_worktree_dir: "", + create_pr: false, + learning_mode: "off", + memory_scope: "none", + }, + ui: { select: async () => "Approve", input: async () => "" }, + task: async (_name: string, options: { output?: string }) => { + if (options.output) await writeMarkdown(options.output, "task output"); + return { text: "draft" }; + }, + workflow: async () => ({ outputs: { result: "implementation done", review_report: structuredEvidenceReport(testCase.evidence), approved: true, status: "complete" } }), + }); + + expect(result.status).toBe(testCase.expectedStatus); + expect(result.approved).toBe(testCase.expectedStatus === "complete"); + if (testCase.expectedMessage) { + expect(result.message).toContain(testCase.expectedMessage); + expect(((result.implementation as Record).gate_review_evidence as Record).decision).toBe("needs_human"); + } + } finally { + await rm(dir, { recursive: true, force: true }); + } + } + }); + test("structured evidence with blocked or conflicted true does not complete", async () => { for (const flag of ["blocked", "conflicted"] as const) { const dir = await mkdtemp(join(tmpdir(), `compound-engineering-${flag}-flag-`)); @@ -1819,7 +2472,7 @@ describe("compound-engineering artifact path helpers and discoverability", () => if (options.output) await writeMarkdown(options.output, "task output"); return { text: "draft" }; }, - workflow: async () => ({ outputs: { result: "implementation done", compound_engineering_evidence: structuredEvidence({ [flag]: true }), approved: true } }), + workflow: async () => ({ outputs: { result: "implementation done", review_report: structuredEvidenceReport(structuredEvidence({ [flag]: true })), approved: true, status: "complete" } }), }); expect(result.status).toBe(flag === "blocked" ? "blocked" : "needs_human"); @@ -1856,10 +2509,11 @@ describe("compound-engineering artifact path helpers and discoverability", () => workflow: async () => ({ outputs: { result: "implementation done", - compound_engineering_evidence: structuredEvidence({ + review_report: structuredEvidenceReport(structuredEvidence({ validation_backed: { satisfied: true, evidence: "Validation was attempted.", commands: [{ command: "bun test", exit_code: 1, summary: "bun test failed" }] }, - }), + })), approved: true, + status: "complete", }, }), }); @@ -1896,13 +2550,13 @@ describe("compound-engineering artifact path helpers and discoverability", () => if (options.output) await writeMarkdown(options.output, "task output"); return { text: "draft" }; }, - workflow: async () => ({ outputs: { result: "implementation done", review_report: proseOnlySufficientReviewReport(), approved: true } }), + workflow: async () => ({ outputs: { result: "implementation done", review_report: proseOnlySufficientReviewReport(), approved: true, status: "complete" } }), }); expect(supplementalTaskRan).toBe(false); expect(result.status).toBe("needs_human"); expect(result.approved).toBe(false); - expect(result.message).toContain("missing named compound_engineering_evidence block"); + expect(result.message).toContain("missing review_evidence object"); expect(result.message).not.toContain("review evidence gate is sufficient"); } finally { await rm(dir, { recursive: true, force: true }); @@ -1943,6 +2597,7 @@ describe("compound-engineering artifact path helpers and discoverability", () => outputs: { result: "implementation done", approved: true, + status: "complete", review_report: sufficientReviewReport(), }, }), @@ -1989,6 +2644,7 @@ describe("compound-engineering artifact path helpers and discoverability", () => outputs: { result: "implementation done", approved: true, + status: "complete", review_report: sufficientReviewReport(), }, }), diff --git a/workflows/compound-engineering/helpers.ts b/workflows/compound-engineering/helpers.ts index b0cd2af..a52faf3 100644 --- a/workflows/compound-engineering/helpers.ts +++ b/workflows/compound-engineering/helpers.ts @@ -221,7 +221,7 @@ export function promptLooksVagueOrProductShaped(prompt: string): boolean { const words = prompt.split(/\s+/).filter(Boolean); const productSignal = /\b(improve|better|onboarding|activation|experience|flow|idea|explore|maybe|help users|make it easier|product|strategy|roadmap)\b/i.test(prompt); const concreteSignal = /\b(api|endpoint|database|schema|migration|test|bug|error|auth|cli|config|typescript|react|sql|cache|worker|specs?\/)\b/i.test(prompt); - return words.length < 8 || (productSignal && !concreteSignal); + return (words.length < 8 && !concreteSignal) || (productSignal && !concreteSignal); } export function resolveMode(promptInput: string, requestedMode: CompoundMode = "auto"): ResolvedCompoundMode { @@ -258,7 +258,19 @@ export function parseApprovalDecision(input: unknown): ApprovalDecision { } function countSeverity(value: unknown): number { - return typeof value === "number" && Number.isFinite(value) && value > 0 ? Math.floor(value) : 0; + return typeof value === "number" && Number.isInteger(value) && value >= 0 ? value : 0; +} + +function hasMalformedSeverityCount(value: unknown): boolean { + return value !== undefined && (typeof value !== "number" || !Number.isInteger(value) || value < 0); +} + +function hasMalformedSeverityCounts(counts: SeverityCounts | undefined): boolean { + if (counts === undefined) return false; + return hasMalformedSeverityCount(counts.p0) + || hasMalformedSeverityCount(counts.p1) + || hasMalformedSeverityCount(counts.p2) + || hasMalformedSeverityCount(counts.p3); } export function normalizeSeverityCounts(counts: SeverityCounts = {}): Required { @@ -284,6 +296,9 @@ export function reduceReviewEvidence(evidence: ReviewEvidence): ReviewEvidenceRe if (evidence.conflicted === true) { return { decision: "needs_human", missing, severity_counts, reason: "Review evidence is conflicted and needs a human decision." }; } + if (hasMalformedSeverityCounts(evidence.severity_counts)) { + return { decision: "needs_human", missing, severity_counts, reason: "Review severity counts are malformed; p0/p1/p2/p3 must be non-negative integers." }; + } if (severity_counts.p0 > 0 || severity_counts.p1 > 0) { return { decision: "fixes_needed", missing, severity_counts, reason: "Blocking P0/P1 findings remain." }; } @@ -382,9 +397,9 @@ export function buildChildHandoff(options: { const childPrompt = [ `Implement the approved Compound Engineering plan/spec at ${options.approvedPath}. Original request: ${options.prompt}`, "", - "After implementation and your own internal review, emit a structured evidence block named `compound_engineering_evidence` in child outputs and/or the `review_report` artifact.", - "The block must reflect the child workflow's own implementation and internal review results; the parent will not fabricate missing evidence or run a duplicative supplemental review.", - "Use exactly this compact JSON-serializable shape:", + "After implementation and your own internal review, return only the child workflow's declared outputs and review artifacts. Do not rely on undeclared parent-specific child output keys.", + "Place structured review evidence in the declared `review_report`/`review_report_path` artifact when the runner supports it; Goal should also return status=complete only when done.", + "The artifact may include a compact JSON object named `review_evidence` with this JSON-serializable shape:", "{", " independent: { satisfied: boolean, evidence?: string, source?: string },", " acceptance_mapped: { satisfied: boolean, evidence?: string },", diff --git a/workflows/compound-engineering/index.ts b/workflows/compound-engineering/index.ts index 327de9f..aa9d5a0 100644 --- a/workflows/compound-engineering/index.ts +++ b/workflows/compound-engineering/index.ts @@ -1,7 +1,17 @@ -import { readFile } from "node:fs/promises"; import { join } from "node:path"; import { defineWorkflow, Type } from "@bastani/workflows"; import { goal, ralph } from "@bastani/workflows/builtin"; +import { + SEVERITY_LEVELS, + emptySeverityCounts, + gateChildRunCompletion, + isRecord, + loadChildReviewArtifact, + optionalNonEmptyString, + parseEvidenceFromText, + type ChildReviewArtifact, + type SeverityLevel, +} from "./lib/review-evidence.js"; import { buildChildHandoff, createArtifactRun, @@ -44,26 +54,36 @@ const DEFAULT_BASE_BRANCH = "origin/main"; const FILE_ONLY_OUTPUT = "file-only" as const; const EVERY_INC_CREDIT = "Inspired by EveryInc's MIT-licensed Compound Engineering Plugin process vocabulary: brainstorm → plan → work → review → compound learning."; -const CHILD_OUTPUT_KEYS = [ +const GOAL_CHILD_OUTPUT_KEYS = [ "result", "status", - "message", - "summary", "approved", - "artifact_dir", - "manifest_path", + "goal_id", + "objective", + "ledger_path", + "turns_completed", + "iterations_completed", + "receipts", + "remaining_work", + "review_report", + "review_report_path", +] as const; + +const RALPH_CHILD_OUTPUT_KEYS = [ + "result", + "plan", "plan_path", "implementation_notes_path", - "ledger_path", "pr_report", + "approved", "iterations_completed", "review_report", "review_report_path", - "validation_output", - "changed_files", - "compound_engineering_evidence", ] as const; -const CHILD_OUTPUT_KEY_SET = new Set(CHILD_OUTPUT_KEYS); + +function declaredChildOutputKeys(runner: Exclude): readonly string[] { + return runner === "goal" ? GOAL_CHILD_OUTPUT_KEYS : RALPH_CHILD_OUTPUT_KEYS; +} const resolvedModeSchema = Type.Union([ Type.Literal("brainstorm"), @@ -98,219 +118,7 @@ function manifestArtifacts(paths: ReadonlyMap, manifestPath: str return artifacts; } -type SufficiencyCriterion = ReviewCriterion; -type SeverityLevel = keyof Required; - -const SEVERITY_LEVELS: readonly SeverityLevel[] = ["p0", "p1", "p2", "p3"]; - -type CriterionPolicy = { - positive: readonly RegExp[]; - negative: readonly RegExp[]; -}; - -const VALIDATION_NO_RUN_PATTERN = /\b(?:validation\s+commands?|tests?|test\s+commands?|commands?)\s+(?:run|ran)\s*:?\s*(?:none|no\s+commands?|zero|0|n\/a)\b|\bno\s+validation\s+(?:(?:was|were)\s+)?(?:run|ran|performed)\b|\bno\s+validation\s+commands?\s+(?:(?:was|were)\s+)?run\b|\bno\s+tests?\s+(?:(?:was|were)\s+)?run\b|\bno\s+commands?\s+(?:(?:was|were)\s+)?run\b|\bvalidation\s+(?:commands?\s+)?skipped\b|\bvalidation\b.{0,50}\bskipped\b|\bvalidation\s+not\s+(?:run|performed|backed|available)\b|\bvalidation\b.{0,50}\b(?:was|were)\s+not\s+(?:run|performed|available)\b|\btests?\b.{0,40}\bskipped\b|\btests?\s+(?:were\s+)?not\s+run\b/; -const CONTEXTUAL_VALIDATION_ERROR_PATTERN = /\b(?:reported\s+)?[1-9]\d*\s+errors\b|\berror\s+count\s*:?\s*[1-9]\d*\b|\berrors\s*:?\s*[1-9]\d*\b|\bfailed\s+with\s+errors\b/; -const VALIDATION_NEGATED_SUCCESS_PATTERN = /\b(?:not|never)\s+(?:(?:all|any|every)\s+(?:[\w-]+\s+){0,6})?(?:pass(?:ed|es|ing)?|successful|success|succeed(?:ed|s|ing)?)\b|\bunsuccessful\b/; -const VALIDATION_SUCCESS_PATTERN = /\b(?:passed|passes|passing|succeeded|successful|success|status\s*:?\s*0|code\s*:?\s*0|exit\s+(?:code|status)\s*:?\s*0|exited\s+with\s+(?:exit\s+)?(?:code|status)\s*:?\s*0|returned\s+(?:exit\s+)?(?:code|status)\s*:?\s*0)\b/; - -const REVIEW_EVIDENCE_POLICIES = { - independent: { - positive: [/\bindependent(?:ly)?\b/, /\bseparate reviewer\b/, /\bfresh[-\s]context reviewer\b/], - negative: [ - /\bnot\s+independent\b/, - /\bno\s+(?:independent|separate|fresh[-\s]context)\s+review(?:er)?\b/, - /\bwithout\s+(?:an?\s+)?(?:independent|separate|fresh[-\s]context)\s+review(?:er)?\b/, - /\bself[-\s]review\b/, - ], - }, - acceptance_mapped: { - positive: [ - /\b(?:map(?:ped)?|mapping|check(?:ed)?|verify|verified|trace(?:d)?|cover(?:ed)?|coverage)\b.{0,80}\b(?:acceptance(?:\s+criteria)?|approved\s+(?:plan|spec))\b/, - /\b(?:acceptance(?:\s+criteria)?|approved\s+(?:plan|spec))\b.{0,80}\b(?:map(?:ped)?|mapping|check(?:ed)?|verify|verified|trace(?:d)?|cover(?:ed)?|coverage)\b/, - ], - negative: [ - /\bno\s+acceptance(?:\s+criteria)?\s+(?:mapping|mapped|traceability|coverage)\b/, - /\bno\s+acceptance(?:\s+criteria)?\s+(?:(?:was|were)\s+)?(?:mapped|checked|verified|traced|covered|inspected)\b/, - /\bnot\s+acceptance[-\s]mapped\b/, - /\bacceptance\s+(?:mapping\s+)?(?:not\s+)?(?:missing|skipped|absent)\b/, - /\b(?:acceptance(?:\s+criteria|\s+mapping)?|approved\s+(?:plan|spec)|spec)\b.{0,80}\b(?:not\s+(?:mapped|provided|performed|done|checked|verified|traced|covered|inspected)|(?:was|were)\s+not\s+(?:mapped|provided|performed|done|checked|verified|traced|covered|inspected)|unmapped|unchecked|unverified|untraced|uncovered|skipped|missing|absent|unavailable)\b/, - /\b(?:did not|does not|not)\s+(?:map|mapped|check|checked|verify|verified|trace|traced|cover|covered|inspect|inspected)\b.{0,80}\b(?:acceptance|approved\s+(?:plan|spec)|spec)\b/, - /\b(?:unchecked|unmapped|unverified|untraced|uncovered)\s+(?:acceptance(?:\s+criteria)?|approved\s+)?(?:spec|plan|criteria)\b/, - ], - }, - diff_aware: { - positive: [/\bdiff\b/, /\bchanged files?\b/, /\bgit status\b/, /\bgit diff\b/], - negative: [ - /\bdiff\s+(?:was\s+|were\s+)?not\s+(?:inspected|reviewed|checked)\b/, - /\bnot\s+(?:inspect(?:ed)?|review(?:ed)?|check(?:ed)?)\b.{0,40}\bdiff\b/, - /\b(?:did not|does not)\s+(?:inspect|review|check)\b.{0,40}\bdiff\b/, - /\bdiff(?:\s+review)?\b.{0,40}\b(?:skipped|missing|absent|not\s+(?:inspected|reviewed|checked|performed)|(?:was|were)\s+not\s+(?:inspected|reviewed|checked|performed))\b/, - /\b(?:unable|could not|cannot)\s+(?:to\s+)?(?:inspect|review|check)\b.{0,60}\b(?:current\s+)?(?:git\s+)?diff\b/, - /\bcould not\s+inspect\s+changed files?\b/, - /\bdiff\s+(?:was\s+)?unavailable\b/, - /\bdiff\s+(?:could\s+not|cannot)\s+be\s+(?:inspected|reviewed|checked)\b/, - /\bdiff\s+was\s+not\s+available\b/, - /\bno\s+(?:git\s+)?diff\s+(?:was\s+)?available\b/, - /\bchanged files?\s+(?:was\s+|were\s+)?(?:unavailable|not\s+(?:inspected|reviewed|checked|available))\b/, - /\bno\s+(?:git\s+)?diff\b.{0,80}\b(?:inspected|reviewed|checked)\b/, - /\bno\s+changed files?\b.{0,80}\b(?:inspected|reviewed|checked)\b/, - ], - }, - validation_backed: { - positive: [ - /\b(?:validation|tests?|commands?|bun\s+test|cargo\s+test|npm\s+test|pnpm\s+test|yarn\s+test|test\s+command)\b.{0,80}\b(?:passed|passes|passing|succeeded|successful|success|status\s*:?\s*0|code\s*:?\s*0|exit\s+(?:code|status)\s*:?\s*0|exited\s+with\s+(?:exit\s+)?(?:code|status)\s*:?\s*0|returned\s+(?:exit\s+)?(?:code|status)\s*:?\s*0)\b/, - /\b(?:passed|passes|passing|succeeded|successful|success|status\s*:?\s*0|code\s*:?\s*0|exit\s+(?:code|status)\s*:?\s*0|exited\s+with\s+(?:exit\s+)?(?:code|status)\s*:?\s*0|returned\s+(?:exit\s+)?(?:code|status)\s*:?\s*0)\b.{0,80}\b(?:validation|tests?|commands?|bun\s+test|cargo\s+test|npm\s+test|pnpm\s+test|yarn\s+test|test\s+command)\b/, - ], - negative: [ - VALIDATION_NO_RUN_PATTERN, - /\b(?:validation|tests?|commands?|bun\s+test|cargo\s+test)\b.{0,80}\b(?:failed|failures?|failing|errored|did\s+not\s+pass|does\s+not\s+pass|not\s+passing|non[-\s]?zero|(?:exit\s+)?(?:code|status)\s*:?\s*[1-9]\d*)\b/, - /\b(?:failed|failures?|failing|errored|did\s+not\s+pass|does\s+not\s+pass|not\s+passing|non[-\s]?zero|(?:exit\s+)?(?:code|status)\s*:?\s*[1-9]\d*)\b.{0,80}\b(?:validation|tests?|commands?|bun\s+test|cargo\s+test)\b/, - /\b(?:validation|tests?|commands?|bun\s+test|cargo\s+test)\b.{0,80}\b(?:reported\s+)?[1-9]\d*\s+errors\b/, - /\b(?:reported\s+)?[1-9]\d*\s+errors\b.{0,80}\b(?:validation|tests?|commands?|bun\s+test|cargo\s+test)\b/, - /\b(?:validation|tests?|commands?|bun\s+test|cargo\s+test)\b.{0,80}\b(?:error\s+count\s*:?\s*[1-9]\d*|errors\s*:?\s*[1-9]\d*|failed\s+with\s+errors)\b/, - /\b(?:error\s+count\s*:?\s*[1-9]\d*|errors\s*:?\s*[1-9]\d*|failed\s+with\s+errors)\b.{0,80}\b(?:validation|tests?|commands?|bun\s+test|cargo\s+test)\b/, - ], - }, - risk_aware: { - positive: [/\brisk\b/, /\bresidual\b/, /\bsecurity\b/], - negative: [ - /\bno\s+residual\s+risk\s+(?:assessment|review|analysis)\b/, - /\bno\s+risk\s+(?:assessment|review|analysis)\b/, - /\brisk\s+(?:assessment|review|analysis)\s+(?:not\s+)?(?:missing|skipped|absent)\b/, - /\brisk\b.{0,50}\bunknown\b/, - /\brisk\b.{0,50}\b(?:not\s+(?:assessed|reviewed|evaluated)|(?:was|were)\s+not\s+(?:assessed|reviewed|evaluated|performed))\b/, - /\bnot\s+(?:risk|residual\s+risk)[-\s]aware\b/, - ], - }, - fresh: { - positive: [ - /\b(?:review(?:ed)?|fresh|evidence)\b.{0,60}\bafter\s+(?:the\s+)?(?:final\s+diff|latest\s+change|current\s+diff)\b/, - /\bafter\s+(?:the\s+)?(?:final\s+diff|latest\s+change|current\s+diff)\b.{0,60}\b(?:review(?:ed)?|fresh|evidence)\b/, - ], - negative: [ - /\bnot\s+fresh\b/, - /\bstale\b/, - /\b(?:review(?:ed)?|evidence)\b.{0,60}\b(?:predates|before)\b.{0,40}\b(?:final\s+diff|latest\s+change|current\s+diff)\b/, - /\b(?:predates|before)\s+(?:the\s+)?(?:final\s+diff|latest\s+change|current\s+diff)\b/, - /\bnot\s+(?:after\s+)?(?:the\s+)?latest\s+change\b/, - /\bcurrent diff\b.{0,60}\bnot\s+(?:inspected|reviewed|checked)\b/, - ], - }, -} as const satisfies Record; - -function reviewSegments(value: string): string[] { - return value - .toLowerCase() - .split(/[\r\n.;]+/) - .map((segment) => segment.trim()) - .filter((segment) => segment.length > 0); -} - -function matchesAnyPattern(value: string, patterns: readonly RegExp[]): boolean { - return patterns.some((pattern) => pattern.test(value)); -} - -function criterionEvidenceState(segments: readonly string[], policy: CriterionPolicy): boolean | undefined { - if (segments.some((segment) => matchesAnyPattern(segment, policy.negative))) return false; - if (segments.some((segment) => matchesAnyPattern(segment, policy.positive))) return true; - return undefined; -} - -const VALIDATION_SUBJECT_PATTERN = /\b(?:validation|tests?|commands?|bun\s+test|cargo\s+test|npm\s+test|pnpm\s+test|yarn\s+test|test\s+command)\b/; -const VALIDATION_RUN_CONTEXT_PATTERN = /\b(?:run|ran|completed|bun\s+test|cargo\s+test|npm\s+test|pnpm\s+test|yarn\s+test|test\s+command)\b/; -const NO_VALIDATION_FAILURE_PATTERN = /\b(?:no|zero|0)\s+(?:validation\s+)?(?:commands?\s+)?(?:failed|failures?|errors?)\b|\b(?:validation\s+)?(?:failures?|errors?)\s*:\s*(?:none|no|zero|0)\b|\b(?:no|zero|0)\s+tests?\s+(?:failed|failing)\b|\bwithout\s+(?:validation\s+|test\s+)?(?:failures?|errors?)\b/g; -const VALIDATION_FAILURE_PATTERN = /\b(?:failed|failures?|failing|errored|did\s+not\s+pass|does\s+not\s+pass|not\s+passing|non[-\s]?zero|(?:exit\s+)?(?:code|status)\s*:?\s*[1-9]\d*|exited\s+with\s+(?:exit\s+)?(?:code|status)\s*:?\s*[1-9]\d*|returned\s+(?:exit\s+)?(?:code|status)\s*:?\s*[1-9]\d*)\b|\b(?:reported\s+)?[1-9]\d*\s+errors\b|\berror\s+count\s*:?\s*[1-9]\d*\b|\berrors\s*:?\s*[1-9]\d*\b|\bfailed\s+with\s+errors\b/; - -function removeNoValidationFailureAssertions(segment: string): string { - return segment.replace(NO_VALIDATION_FAILURE_PATTERN, " ").trim(); -} - -function hasValidationRunContext(segment: string): boolean { - return VALIDATION_SUBJECT_PATTERN.test(segment) && VALIDATION_RUN_CONTEXT_PATTERN.test(segment) && !VALIDATION_NO_RUN_PATTERN.test(segment); -} - -function validationConditionHasContext(segment: string, previousSegment: string): boolean { - return VALIDATION_SUBJECT_PATTERN.test(segment) || (previousSegment.length > 0 && hasValidationRunContext(previousSegment)); -} - -function hasValidationFailure(segments: readonly string[]): boolean { - for (let index = 0; index < segments.length; index += 1) { - const scrubbed = removeNoValidationFailureAssertions(segments[index]); - if (scrubbed.length === 0) continue; - - const hasFailure = VALIDATION_FAILURE_PATTERN.test(scrubbed) || CONTEXTUAL_VALIDATION_ERROR_PATTERN.test(scrubbed); - const hasNegatedSuccess = VALIDATION_NEGATED_SUCCESS_PATTERN.test(scrubbed); - if (!hasFailure && !hasNegatedSuccess) continue; - - const previous = index > 0 ? removeNoValidationFailureAssertions(segments[index - 1]) : ""; - if (validationConditionHasContext(scrubbed, previous)) return true; - } - return false; -} - -function validationEvidenceState(segments: readonly string[]): boolean | undefined { - const scrubbedSegments = segments - .map(removeNoValidationFailureAssertions) - .filter((segment) => segment.length > 0); - const validationPolicy = REVIEW_EVIDENCE_POLICIES.validation_backed; - - for (let index = 0; index < scrubbedSegments.length; index += 1) { - const segment = scrubbedSegments[index]; - if (!VALIDATION_NEGATED_SUCCESS_PATTERN.test(segment)) continue; - - const previous = index > 0 ? scrubbedSegments[index - 1] : ""; - if (validationConditionHasContext(segment, previous)) return false; - } - - if (scrubbedSegments.some((segment) => matchesAnyPattern(segment, validationPolicy.negative))) return false; - if (scrubbedSegments.some((segment) => matchesAnyPattern(segment, validationPolicy.positive))) return true; - - for (let index = 1; index < scrubbedSegments.length; index += 1) { - if (VALIDATION_SUCCESS_PATTERN.test(scrubbedSegments[index]) && hasValidationRunContext(scrubbedSegments[index - 1])) return true; - } - return undefined; -} - -function emptySeverityCounts(): Required { - return { p0: 0, p1: 0, p2: 0, p3: 0 }; -} - -function severityFindingLines(value: string, severity: SeverityLevel): number { - const findingPattern = new RegExp(`(?:^|\\s)(?:-\\s*)?(?:\\[${severity}\\]|${severity}\\s*(?::\\s*\\S|[-–—]\\s*\\S))`, "i"); - return value - .split(/\r?\n/) - .map((line) => line.trim()) - .filter((line) => { - if (line.length === 0) return false; - if (/^#{1,6}\s*(?:p[0-3])\b/i.test(line)) return false; - if (/^(?:-\s*)?(?:p[0-3]\s*:\s*)?(?:none(?:\s+(?:found|identified|reported))?|no\s+(?:findings?|issues?|blockers?)|n\/a|nothing)\.?$/i.test(line)) return false; - return findingPattern.test(line); - }) - .length; -} - -export function parseEvidenceFromText(value: string): ReviewEvidence { - const textValue = value.toLowerCase(); - const segments = reviewSegments(value); - const validationFailed = hasValidationFailure(segments); - const severity_counts = emptySeverityCounts(); - for (const severity of SEVERITY_LEVELS) { - severity_counts[severity] = severityFindingLines(value, severity); - } - return { - independent: criterionEvidenceState(segments, REVIEW_EVIDENCE_POLICIES.independent), - acceptance_mapped: criterionEvidenceState(segments, REVIEW_EVIDENCE_POLICIES.acceptance_mapped), - diff_aware: criterionEvidenceState(segments, REVIEW_EVIDENCE_POLICIES.diff_aware), - validation_backed: validationFailed ? false : validationEvidenceState(segments), - risk_aware: criterionEvidenceState(segments, REVIEW_EVIDENCE_POLICIES.risk_aware), - fresh: criterionEvidenceState(segments, REVIEW_EVIDENCE_POLICIES.fresh), - severity_counts, - conflicted: /conflict(ed|ing)? evidence/.test(textValue), - validation_failed: validationFailed, - blocked: /blocked|unable to review|missing dependency/.test(textValue), - }; -} function childOutputs(result: unknown): Record { if (typeof result !== "object" || result === null) return {}; @@ -354,21 +162,71 @@ function textFromUnknown(value: unknown): string { } } -function normalizedChildOutput(result: unknown): Record { +function normalizedChildOutput(result: unknown, runner: Exclude): Record { const outputs = childOutputs(result); + const declaredKeys = declaredChildOutputKeys(runner); + const declaredKeySet = new Set(declaredKeys); const selected: Record = {}; - for (const key of CHILD_OUTPUT_KEYS) { + for (const key of declaredKeys) { if (outputs[key] !== undefined) selected[key] = outputs[key]; } - const omittedChildOutputKeys = Object.keys(outputs) - .filter((key) => !CHILD_OUTPUT_KEY_SET.has(key)) + const unknownChildOutputKeys = Object.keys(outputs) + .filter((key) => !declaredKeySet.has(key)) .sort(); - if (omittedChildOutputKeys.length > 0) selected.omitted_child_output_keys = omittedChildOutputKeys; + if (unknownChildOutputKeys.length > 0) selected.unknown_child_output_keys = unknownChildOutputKeys; return outputRecord(selected); } +function childWorkflowExited(result: unknown): boolean { + return typeof result === "object" && result !== null && (result as { exited?: unknown }).exited === true; +} + +type ChildAtomicExitStatus = "blocked" | "cancelled" | "skipped"; + +function childExitRecord(result: unknown): Record { + return typeof result === "object" && result !== null ? result as Record : {}; +} + +function childExitStatus(result: unknown): ChildAtomicExitStatus { + const record = childExitRecord(result); + const nestedExit = isRecord(record.exit) ? record.exit : undefined; + const rawStatus = optionalNonEmptyString(record.status) + ?? optionalNonEmptyString(record.exit_status) + ?? optionalNonEmptyString(nestedExit?.status); + const normalized = rawStatus?.toLowerCase(); + if (normalized === "cancelled" || normalized === "canceled") return "cancelled"; + if (normalized === "skipped") return "skipped"; + return "blocked"; +} + +function childExitReason(result: unknown, fallback: string): string { + const record = childExitRecord(result); + const nestedExit = isRecord(record.exit) ? record.exit : undefined; + return optionalNonEmptyString(record.exitReason) + ?? optionalNonEmptyString(record.reason) + ?? optionalNonEmptyString(record.exit_reason) + ?? optionalNonEmptyString(nestedExit?.reason) + ?? fallback; +} + +function domainStatusForChildExit(exitStatus: ChildAtomicExitStatus): WorkflowStatus { + return exitStatus === "cancelled" || exitStatus === "skipped" ? "stopped" : "blocked"; +} + +function exitWorkflow(ctx: unknown, outputs: Record, exitStatus: unknown = outputs.status, exitReason: unknown = outputs.message): Record { + const exit = typeof ctx === "object" && ctx !== null ? (ctx as { exit?: unknown }).exit : undefined; + if (typeof exit === "function") { + return exit.call(ctx, { + status: exitStatus, + reason: text(exitReason), + outputs, + }) as Record; + } + return outputs; +} + const MAX_RECEIPT_STRING_CHARS = 2000; const MAX_RECEIPT_COLLECTION_ITEMS = 50; const MAX_RECEIPT_OBJECT_KEYS = 50; @@ -424,288 +282,26 @@ function compactReceiptValue(value: unknown, depth = 0, seen = new WeakSet): Record { const compacted: Record = {}; + const omittedInlineChildOutputKeys: string[] = []; for (const [key, value] of Object.entries(childOutput)) { - compacted[key] = compactReceiptValue(value); - } - return outputRecord(compacted); -} - -type ChildRunGate = - | { state: "approved"; parent_status?: undefined; reason: string; approved: true; status?: string } - | { state: "non_approved"; parent_status: "needs_human"; reason: string; approved?: boolean; status?: string } - | { state: "blocked"; parent_status: "blocked"; reason: string; approved?: boolean; status: string } - | { state: "missing_approval"; parent_status: "needs_human"; reason: string; approved?: boolean; status?: string }; - -type ChildReviewArtifact = { - body: string; - evidenceText: string; - trace: Record; - reportPath?: string; -}; - -function optionalBoolean(value: unknown): boolean | undefined { - return typeof value === "boolean" ? value : undefined; -} - -function optionalNonEmptyString(value: unknown): string | undefined { - const stringValue = typeof value === "string" ? value.trim() : ""; - return stringValue.length > 0 ? stringValue : undefined; -} - -function errorMessage(error: unknown): string { - return error instanceof Error ? error.message : String(error); -} - -const CHILD_FAILURE_STATUSES = new Set(["blocked", "failed", "failure", "error", "errored"]); -const CHILD_NEEDS_HUMAN_STATUSES = new Set(["needs_human", "rejected", "stopped", "active", "pending", "running"]); -const CHILD_SUCCESS_STATUSES = new Set(["complete", "completed", "success", "succeeded", "passed"]); - -function childStatusIs(status: string | undefined, statuses: ReadonlySet): boolean { - return status !== undefined && statuses.has(status); -} - -export function gateChildRunCompletion(childOutput: Record, runner: Exclude): ChildRunGate { - const approved = optionalBoolean(childOutput.approved); - const status = optionalNonEmptyString(childOutput.status); - const normalizedStatus = status?.toLowerCase(); - - if (childStatusIs(normalizedStatus, CHILD_FAILURE_STATUSES)) { - return { state: "blocked", parent_status: "blocked", reason: `Child ${runner} returned status=${status}.`, approved, status }; - } - if (approved === false) { - return { state: "non_approved", parent_status: "needs_human", reason: `Child ${runner} returned approved=false.`, approved, status }; - } - if (childStatusIs(normalizedStatus, CHILD_NEEDS_HUMAN_STATUSES)) { - return { state: "non_approved", parent_status: "needs_human", reason: `Child ${runner} returned status=${status}.`, approved, status }; - } - if (approved !== true) { - return { state: "missing_approval", parent_status: "needs_human", reason: `Child ${runner} did not return approved=true.`, approved, status }; - } - if (normalizedStatus !== undefined && !childStatusIs(normalizedStatus, CHILD_SUCCESS_STATUSES)) { - return { state: "non_approved", parent_status: "needs_human", reason: `Child ${runner} returned unknown status=${status}.`, approved, status }; - } - - return { state: "approved", reason: `Child ${runner} returned approved=true.`, approved, status }; -} - -type CompactReviewPointer = { - kind: "output_saved_to" | "latest_review_round_artifact"; - path?: string; -}; - -const ATOMIC_FILE_ONLY_SUFFIX_PATTERN = /\s+\(\d+(?:\.\d+)?\s*(?:B|KB|MB|GB),\s*\d+\s+lines?\)\.?(?:\s+Read this file if needed\.?)?$/i; - -function unquoteMatchingPath(value: string): string { - const quote = value[0]; - if ((quote === '"' || quote === "'") && value[value.length - 1] === quote) return value.slice(1, -1); - return value; -} - -function compactReviewPointerKind(label: string): CompactReviewPointer["kind"] { - return /^latest review round artifact$/i.test(label) ? "latest_review_round_artifact" : "output_saved_to"; -} - -function parseCompactReviewPointer(value: string): CompactReviewPointer | undefined { - const firstSubstantiveLine = value - .split(/\r?\n/) - .map((line) => line.trim()) - .find((line) => line.length > 0) ?? ""; - - const match = firstSubstantiveLine.match(/^(output saved to|saved output to|saved to|latest review round artifact):\s*(.*)$/i); - if (!match) return undefined; - - const kind = compactReviewPointerKind(match[1]); - const target = match[2].trim(); - if (target.length === 0) return { kind }; - - const path = unquoteMatchingPath(target.replace(ATOMIC_FILE_ONLY_SUFFIX_PATTERN, "").trim()); - return { kind, path }; -} - -function appendMultilineReviewArtifactField(lines: string[], label: string, value: string): void { - lines.push(`${label}:`); - lines.push(...value - .replace(/\.\s+(?=P[0-3]\s*:)/gi, ".\n") - .split(/\r?\n/) - .map((line) => line.trim()) - .filter((line) => line.length > 0)); -} - -function appendReviewArtifactField(lines: string[], label: string, value: unknown): void { - if (typeof value === "number" || typeof value === "boolean") { - lines.push(`${label}: ${String(value)}`); - return; - } - if (typeof value !== "string") return; - - const trimmed = value.trim(); - if (trimmed.length === 0) return; - - if (label === "raw_text" || label === "overall_explanation") { - appendMultilineReviewArtifactField(lines, label, trimmed); - } else { - lines.push(`${label}: ${trimmed}`); - } - - if (label === "overall_correctness" && /\b(?:incorrect|wrong|not\s+correct|failed|failure)\b/i.test(trimmed)) { - lines.push(`P1: overall_correctness ${trimmed}`); - } -} - -function prioritySeverity(value: unknown): SeverityLevel | undefined { - if (typeof value === "number" && Number.isInteger(value) && value >= 0 && value <= 3) return `p${value}` as SeverityLevel; - const match = typeof value === "string" ? value.match(/\bp([0-3])\b|^\s*([0-3])\s*$/i) : undefined; - const digit = match?.[1] ?? match?.[2]; - return digit ? `p${digit}` as SeverityLevel : undefined; -} - -function appendReviewFinding(lines: string[], finding: Record): void { - const title = optionalNonEmptyString(finding.title); - const body = optionalNonEmptyString(finding.body) ?? optionalNonEmptyString(finding.description); - const priority = prioritySeverity(finding.priority ?? finding.severity); - - if (priority !== undefined) lines.push(`${priority.toUpperCase()}: ${title ?? body ?? "structured finding"}`); - if (title !== undefined) lines.push(`finding_title: ${title}`); - if (body !== undefined) lines.push(`finding_body: ${body}`); - appendReviewArtifactField(lines, "priority", finding.priority ?? finding.severity); - - const codeLocation = finding.code_location; - if (typeof codeLocation === "object" && codeLocation !== null) { - const path = optionalNonEmptyString((codeLocation as Record).absolute_file_path); - if (path !== undefined) lines.push(`file: ${path}`); - } -} - -function appendSeverityCounts(lines: string[], value: Record): void { - for (const severity of SEVERITY_LEVELS) { - const count = value[severity] ?? value[severity.toUpperCase()]; - if (typeof count === "number" && Number.isFinite(count) && count > 0) { - lines.push(`${severity.toUpperCase()}: severity count reported ${Math.floor(count)}`); - } else if (typeof count === "number" && count === 0) { - lines.push(`${severity.toUpperCase()}: none`); - } - } -} - -function appendReviewArtifactJson(lines: string[], value: unknown, key = "root"): void { - if (Array.isArray(value)) { - for (const item of value) appendReviewArtifactJson(lines, item, key); - return; - } - if (typeof value !== "object" || value === null) return; - - const record = value as Record; - if (key === "findings") { - appendReviewFinding(lines, record); - } - - for (const [childKey, childValue] of Object.entries(record)) { - if (childKey === "findings" && Array.isArray(childValue)) { - for (const finding of childValue) { - if (typeof finding === "object" && finding !== null) appendReviewFinding(lines, finding as Record); - } + if (OMITTED_INLINE_RECEIPT_OUTPUT_KEYS.has(key)) { + omittedInlineChildOutputKeys.push(key); continue; } - if (childKey === "severity_counts" && typeof childValue === "object" && childValue !== null) { - appendSeverityCounts(lines, childValue as Record); - continue; - } - if (/^(?:reviewer|overall_correctness|overall_explanation|raw_text|validation(?:_notes|_output|_summary|_results)?|commands_run|notes?)$/i.test(childKey)) { - appendReviewArtifactField(lines, childKey, childValue); - continue; - } - appendReviewArtifactJson(lines, childValue, childKey); - } -} - -export function reviewArtifactToEvidenceText(body: string): string { - try { - const parsed = JSON.parse(body) as unknown; - const lines: string[] = []; - appendReviewArtifactJson(lines, parsed); - return lines.length > 0 ? lines.join("\n") : body; - } catch { - return body; + compacted[key] = compactReceiptValue(value); } + if (omittedInlineChildOutputKeys.length > 0) compacted.omitted_inline_child_output_keys = omittedInlineChildOutputKeys.sort(); + return outputRecord(compacted); } -async function loadReviewArtifactPath(path: string): Promise { - const body = await readFile(path, "utf8"); - if (body.trim().length === 0) throw new Error(`Child review artifact is empty: ${path}`); - return body; -} - -async function loadReviewEvidenceArtifact(path: string, trace: Record): Promise { - const body = await loadReviewArtifactPath(path); - return { - body, - evidenceText: reviewArtifactToEvidenceText(body), - reportPath: path, - trace, - }; -} - -export async function loadChildReviewArtifact(childOutput: Record): Promise { - const reviewReportPath = optionalNonEmptyString(childOutput.review_report_path); - const inlineReviewReport = optionalNonEmptyString(childOutput.review_report); - const inlineCompactPointer = inlineReviewReport !== undefined ? parseCompactReviewPointer(inlineReviewReport) : undefined; - - if (reviewReportPath !== undefined) { - try { - return await loadReviewEvidenceArtifact(reviewReportPath, { source: "review_report_path", path: reviewReportPath, loaded: true }); - } catch (error) { - const reviewReportPathError = errorMessage(error); - return { - body: "", - evidenceText: `blocked unable to review missing dependency: child review_report_path unreadable: ${reviewReportPath}: ${reviewReportPathError}`, - trace: { source: "review_report_path", path: reviewReportPath, loaded: false, error: reviewReportPathError, fail_closed: true }, - }; - } - } - - if (inlineReviewReport !== undefined) { - if (inlineCompactPointer !== undefined) { - if (inlineCompactPointer.path !== undefined) { - try { - return await loadReviewEvidenceArtifact(inlineCompactPointer.path, { - source: "inline_review_report_pointer", - path: inlineCompactPointer.path, - loaded: true, - compact_pointer: true, - pointer_kind: inlineCompactPointer.kind, - }); - } catch (error) { - const pointerErrorMessage = errorMessage(error); - return { - body: "", - evidenceText: `blocked unable to review missing dependency: compact child review_report pointer unreadable: ${inlineCompactPointer.path}: ${pointerErrorMessage}`, - trace: { source: "inline_review_report_pointer", path: inlineCompactPointer.path, loaded: false, compact_pointer: true, pointer_kind: inlineCompactPointer.kind, error: pointerErrorMessage, fail_closed: true }, - }; - } - } - return { - body: "", - evidenceText: "blocked unable to review missing dependency: child review_report is only a compact file-only pointer without a readable path", - trace: { source: "inline_review_report_pointer", loaded: false, compact_pointer: true, pointer_kind: inlineCompactPointer.kind, fail_closed: true }, - }; - } - return { - body: inlineReviewReport, - evidenceText: reviewArtifactToEvidenceText(inlineReviewReport), - trace: { source: "inline_review_report", loaded: true }, - }; - } - return { - body: "", - evidenceText: "blocked unable to review missing dependency: child output did not include review_report_path or substantive review_report", - trace: { source: "missing_child_review_report", loaded: false, fail_closed: true }, - }; -} -const COMPOUND_ENGINEERING_EVIDENCE_KEY = "compound_engineering_evidence"; +const LEGACY_COMPOUND_ENGINEERING_EVIDENCE_KEY = "compound_engineering_evidence"; +const DECLARED_REVIEW_EVIDENCE_KEYS = ["review_evidence", "compound_review_evidence"] as const; type StructuredEvidenceCommand = { command?: string; @@ -741,9 +337,7 @@ type StructuredEvidenceGate = { childReviewArtifact?: ChildReviewArtifact; }; -function isRecord(value: unknown): value is Record { - return typeof value === "object" && value !== null && !Array.isArray(value); -} + function compactErrorList(values: readonly string[]): string[] { return [...new Set(values)].slice(0, 20); @@ -754,7 +348,7 @@ function hasSubstantiveEvidence(value: unknown): boolean { } const PASSING_COMMAND_SUMMARY_PATTERN = /\b(?:pass(?:ed|es|ing)?|succeed(?:ed|s|ing)?|success(?:ful|fully)?|exit\s+(?:code|status)\s*:?\s*0|(?:exit_)?code\s*:?\s*0|status\s*:?\s*0|0\s+(?:failures?|errors?)|zero\s+(?:failures?|errors?)|no\s+(?:failures?|errors?)|all\s+tests?\s+passed)\b/i; -const FAILING_COMMAND_SUMMARY_PATTERN = /\b(?:fail(?:ed|s|ing|ures?)|errored|non[-\s]?zero|exit\s+(?:code|status)\s*:?\s*[1-9]\d*|(?:exit_)?code\s*:?\s*[1-9]\d*|status\s*:?\s*[1-9]\d*)\b/i; +const FAILING_COMMAND_SUMMARY_PATTERN = /\b(?:failed|fails|failing|errored|non[-\s]?zero|exit\s+(?:code|status)\s*:?\s*[1-9]\d*|(?:exit_)?code\s*:?\s*[1-9]\d*|status\s*:?\s*[1-9]\d*|[1-9]\d*\s+(?:failures?|errors?)|(?:failures?|errors?)\s*:?\s*[1-9]\d*)\b/i; function commandSummaryPasses(summary: unknown): boolean { if (typeof summary !== "string") return false; @@ -767,9 +361,9 @@ function validationCommandsPass(commands: unknown): boolean { if (!Array.isArray(commands)) return false; for (const command of commands) { if (!isRecord(command)) return false; - const exitCode = command.exit_code; - if (typeof exitCode === "number" && Number.isFinite(exitCode)) { - if (Math.trunc(exitCode) !== 0) return false; + if (Object.prototype.hasOwnProperty.call(command, "exit_code")) { + const exitCode = command.exit_code; + if (typeof exitCode !== "number" || !Number.isInteger(exitCode) || exitCode !== 0) return false; continue; } if (!commandSummaryPasses(command.summary)) return false; @@ -791,11 +385,11 @@ function readSeverityCounts(value: unknown, errors: string[]): Required> = {}; for (const severity of SEVERITY_LEVELS) { const count = value[severity]; - if (typeof count !== "number" || !Number.isFinite(count) || count < 0) { - errors.push(`severity_counts.${severity} must be a non-negative number`); + if (typeof count !== "number" || !Number.isInteger(count) || count < 0) { + errors.push(`severity_counts.${severity} must be a non-negative integer`); return undefined; } - counts[severity] = Math.floor(count); + counts[severity] = count; } return counts as Required; @@ -810,7 +404,7 @@ function structuredEvidenceToReviewEvidence(value: unknown): { evidence: ReviewE return { evidence, missing: [...REVIEW_CRITERIA], - errors: [`${COMPOUND_ENGINEERING_EVIDENCE_KEY} must be an object`], + errors: ["review_evidence must be an object"], }; } @@ -846,9 +440,14 @@ function structuredEvidenceToReviewEvidence(value: unknown): { evidence: ReviewE const severity_counts = readSeverityCounts(value.severity_counts, errors); if (severity_counts !== undefined) evidence.severity_counts = severity_counts; - if (typeof value.blocked === "boolean") evidence.blocked = value.blocked; - if (typeof value.conflicted === "boolean") evidence.conflicted = value.conflicted; - if (typeof value.validation_failed === "boolean") evidence.validation_failed = value.validation_failed; + for (const flag of ["blocked", "conflicted", "validation_failed"] as const) { + if (!Object.prototype.hasOwnProperty.call(value, flag)) continue; + if (typeof value[flag] === "boolean") { + evidence[flag] = value[flag]; + } else { + errors.push(`${flag} must be boolean when present`); + } + } if (isRecord(value.validation_backed) && validationCommandsPass(value.validation_backed.commands) === false) { evidence.validation_backed = false; evidence.validation_failed = true; @@ -859,21 +458,51 @@ function structuredEvidenceToReviewEvidence(value: unknown): { evidence: ReviewE return { evidence, missing, errors: compactErrorList(errors) }; } -function findNamedEvidence(value: unknown, seen = new WeakSet()): unknown | undefined { - if (!isRecord(value) && !Array.isArray(value)) return undefined; - if (seen.has(value)) return undefined; +type JsonEvidenceExtraction = { + evidence?: unknown; + legacyEvidenceFound: boolean; +}; + +function valueLooksLikeReviewEvidence(value: unknown): boolean { + if (!isRecord(value)) return false; + return REVIEW_CRITERIA.some((criterion) => isRecord(value[criterion])) + || isRecord(value.severity_counts); +} + +function findDeclaredArtifactEvidence(value: unknown, seen = new WeakSet()): JsonEvidenceExtraction { + if (!isRecord(value) && !Array.isArray(value)) return { legacyEvidenceFound: false }; + if (seen.has(value)) return { legacyEvidenceFound: false }; seen.add(value); - if (isRecord(value) && Object.prototype.hasOwnProperty.call(value, COMPOUND_ENGINEERING_EVIDENCE_KEY)) { - return value[COMPOUND_ENGINEERING_EVIDENCE_KEY]; + let legacyEvidenceFound = false; + if (isRecord(value)) { + if (Object.prototype.hasOwnProperty.call(value, LEGACY_COMPOUND_ENGINEERING_EVIDENCE_KEY)) legacyEvidenceFound = true; + for (const key of DECLARED_REVIEW_EVIDENCE_KEYS) { + if (Object.prototype.hasOwnProperty.call(value, key)) { + return { evidence: value[key], legacyEvidenceFound }; + } + } + if (valueLooksLikeReviewEvidence(value)) return { evidence: value, legacyEvidenceFound }; } - const values = Array.isArray(value) ? value : Object.values(value); - for (const childValue of values) { - const found = findNamedEvidence(childValue, seen); - if (found !== undefined) return found; + if (Array.isArray(value)) { + for (const childValue of value) { + const found = findDeclaredArtifactEvidence(childValue, seen); + legacyEvidenceFound = legacyEvidenceFound || found.legacyEvidenceFound; + if (found.evidence !== undefined) return { evidence: found.evidence, legacyEvidenceFound }; + } + } else { + for (const [key, childValue] of Object.entries(value)) { + if (key === LEGACY_COMPOUND_ENGINEERING_EVIDENCE_KEY) { + legacyEvidenceFound = true; + continue; + } + const found = findDeclaredArtifactEvidence(childValue, seen); + legacyEvidenceFound = legacyEvidenceFound || found.legacyEvidenceFound; + if (found.evidence !== undefined) return { evidence: found.evidence, legacyEvidenceFound }; + } } - return undefined; + return { legacyEvidenceFound }; } function parseJsonCandidate(value: string): unknown | undefined { @@ -884,21 +513,191 @@ function parseJsonCandidate(value: string): unknown | undefined { } } -function extractJsonEvidenceFromBody(body: string): unknown | undefined { +type NativeReviewRoundGate = { + childEvidence: ReviewEvidence; + trace: Record; + missing: ReviewCriterion[]; + errors: string[]; +}; + +function nativeReviewEvidence(severity_counts: Required): ReviewEvidence { + return { + independent: true, + acceptance_mapped: true, + diff_aware: true, + validation_backed: true, + risk_aware: true, + fresh: true, + severity_counts, + }; +} + +function isNativeReviewRoundCandidate(value: unknown): value is Record { + return isRecord(value) + && Object.prototype.hasOwnProperty.call(value, "reviews") + && (Object.prototype.hasOwnProperty.call(value, "turn") || Object.prototype.hasOwnProperty.call(value, "iteration")); +} + +function reviewerErrorIsPresent(value: unknown): boolean { + if (value === undefined || value === null || value === false) return false; + return !(typeof value === "string" && value.trim().length === 0); +} + +function nativeFindingSeverity(value: unknown): SeverityLevel | undefined { + if (typeof value === "number" && Number.isInteger(value) && value >= 0 && value <= 3) return `p${value}` as SeverityLevel; + if (typeof value !== "string") return undefined; + const match = value.trim().match(/^(?:p)?([0-3])$/i); + return match ? `p${match[1]}` as SeverityLevel : undefined; +} + +function readNativeDecisionFindings(value: unknown, errors: string[], reviewIndex: number): Required { + const counts = emptySeverityCounts(); + if (value === undefined) return counts; + if (!Array.isArray(value)) { + errors.push(`reviews[${reviewIndex}].decision.findings must be an array when present`); + return counts; + } + + for (let findingIndex = 0; findingIndex < value.length; findingIndex += 1) { + const finding = value[findingIndex]; + if (!isRecord(finding)) { + errors.push(`reviews[${reviewIndex}].decision.findings[${findingIndex}] must be an object`); + continue; + } + const severityValue = finding.priority ?? finding.severity; + const severity = nativeFindingSeverity(severityValue); + if (severity === undefined) { + errors.push(`reviews[${reviewIndex}].decision.findings[${findingIndex}] has unknown or malformed severity`); + continue; + } + counts[severity] += 1; + } + return counts; +} + +function mergeSeverityCounts(target: Required, source: Required): void { + for (const severity of SEVERITY_LEVELS) target[severity] += source[severity]; +} + +function findNativeReviewRound(value: unknown, seen = new WeakSet()): Record | undefined { + if (!isRecord(value) && !Array.isArray(value)) return undefined; + if (seen.has(value)) return undefined; + seen.add(value); + + if (isNativeReviewRoundCandidate(value)) return value; + if (Array.isArray(value)) { + for (const childValue of value) { + const found = findNativeReviewRound(childValue, seen); + if (found !== undefined) return found; + } + } else { + for (const [key, childValue] of Object.entries(value)) { + if (key === LEGACY_COMPOUND_ENGINEERING_EVIDENCE_KEY) continue; + const found = findNativeReviewRound(childValue, seen); + if (found !== undefined) return found; + } + } + return undefined; +} + +function parseNativeReviewRound(value: Record, runner: Exclude): NativeReviewRoundGate { + const errors: string[] = []; + const severity_counts = emptySeverityCounts(); + const sourceKind = runner === "goal" ? "native_goal_review_round" : "native_ralph_review_round"; + + if (Object.prototype.hasOwnProperty.call(value, "stop_review_loop") && value.stop_review_loop !== true) { + errors.push("stop_review_loop must be true when present"); + } + if (reviewerErrorIsPresent(value.reviewer_error)) { + errors.push("native review round contains reviewer_error"); + } + + if (!Array.isArray(value.reviews) || value.reviews.length === 0) { + errors.push("native review round must include a non-empty reviews array"); + } else { + for (let reviewIndex = 0; reviewIndex < value.reviews.length; reviewIndex += 1) { + const review = value.reviews[reviewIndex]; + if (!isRecord(review)) { + errors.push(`reviews[${reviewIndex}] must be an object`); + continue; + } + if (optionalNonEmptyString(review.reviewer) === undefined) { + errors.push(`reviews[${reviewIndex}].reviewer must be a non-empty string`); + } + if (reviewerErrorIsPresent(review.reviewer_error)) { + errors.push(`reviews[${reviewIndex}] contains reviewer_error`); + } + if (Object.prototype.hasOwnProperty.call(review, "stop_review_loop") && review.stop_review_loop !== true) { + errors.push(`reviews[${reviewIndex}].stop_review_loop must be true when present`); + } + const decision = review.decision; + if (!isRecord(decision)) { + errors.push(`reviews[${reviewIndex}].decision must be an object`); + continue; + } + if (reviewerErrorIsPresent(decision.reviewer_error)) { + errors.push(`reviews[${reviewIndex}].decision contains reviewer_error`); + } + if (Object.prototype.hasOwnProperty.call(decision, "stop_review_loop") && decision.stop_review_loop !== true) { + errors.push(`reviews[${reviewIndex}].decision.stop_review_loop must be true when present`); + } + if (typeof decision.overall_correctness !== "string" || decision.overall_correctness.trim().toLowerCase() !== "patch is correct") { + errors.push(`reviews[${reviewIndex}].decision.overall_correctness must be \"patch is correct\"`); + } + mergeSeverityCounts(severity_counts, readNativeDecisionFindings(decision.findings, errors, reviewIndex)); + } + } + + const childEvidence = nativeReviewEvidence(severity_counts); + return { + childEvidence, + trace: { + contract: "native_builtin_review_round", + source_kind: sourceKind, + loaded: errors.length === 0, + turn: value.turn, + iteration: value.iteration, + review_count: Array.isArray(value.reviews) ? value.reviews.length : 0, + missing: errors.length > 0 ? [...REVIEW_CRITERIA] : [], + errors: compactErrorList(errors), + }, + missing: errors.length > 0 ? [...REVIEW_CRITERIA] : [], + errors: compactErrorList(errors), + }; +} + +function extractNativeReviewRoundFromBody(body: string, runner: Exclude): NativeReviewRoundGate | undefined { const parsedBody = parseJsonCandidate(body); - const direct = parsedBody !== undefined ? findNamedEvidence(parsedBody) : undefined; - if (direct !== undefined) return direct; + const directNative = parsedBody !== undefined ? findNativeReviewRound(parsedBody) : undefined; + if (directNative !== undefined) return parseNativeReviewRound(directNative, runner); const fencePattern = /```(?:json)?\s*([\s\S]*?)```/gi; for (const match of body.matchAll(fencePattern)) { const parsedFence = parseJsonCandidate(match[1].trim()); - const fromFence = parsedFence !== undefined ? findNamedEvidence(parsedFence) : undefined; - if (fromFence !== undefined) return fromFence; + const fencedNative = parsedFence !== undefined ? findNativeReviewRound(parsedFence) : undefined; + if (fencedNative !== undefined) return parseNativeReviewRound(fencedNative, runner); } return undefined; } +function extractJsonEvidenceFromBody(body: string): JsonEvidenceExtraction { + const parsedBody = parseJsonCandidate(body); + const direct = parsedBody !== undefined ? findDeclaredArtifactEvidence(parsedBody) : { legacyEvidenceFound: false }; + if (direct.evidence !== undefined) return direct; + + let legacyEvidenceFound = direct.legacyEvidenceFound; + const fencePattern = /```(?:json)?\s*([\s\S]*?)```/gi; + for (const match of body.matchAll(fencePattern)) { + const parsedFence = parseJsonCandidate(match[1].trim()); + const fromFence = parsedFence !== undefined ? findDeclaredArtifactEvidence(parsedFence) : { legacyEvidenceFound: false }; + legacyEvidenceFound = legacyEvidenceFound || fromFence.legacyEvidenceFound; + if (fromFence.evidence !== undefined) return { evidence: fromFence.evidence, legacyEvidenceFound }; + } + + return { legacyEvidenceFound }; +} + function structuredEvidenceNeedsHumanReduction(gate: StructuredEvidenceGate, fallback: ReviewEvidenceReduction): ReviewEvidenceReduction { if (fallback.decision === "sufficient" && gate.errors.length > 0) { return { @@ -911,31 +710,14 @@ function structuredEvidenceNeedsHumanReduction(gate: StructuredEvidenceGate, fal return fallback; } -async function loadStructuredChildEvidence(childOutput: Record): Promise { - const directEvidence = childOutput[COMPOUND_ENGINEERING_EVIDENCE_KEY]; - if (directEvidence !== undefined) { - const converted = structuredEvidenceToReviewEvidence(directEvidence); - return { - childEvidence: converted.evidence, - trace: { - source: "child_output", - contract: COMPOUND_ENGINEERING_EVIDENCE_KEY, - loaded: true, - missing: converted.missing, - errors: converted.errors, - }, - missing: converted.missing, - errors: converted.errors, - }; - } - +async function loadStructuredChildEvidence(childOutput: Record, runner: Exclude): Promise { const childReviewArtifact = await loadChildReviewArtifact(childOutput); if (childReviewArtifact.trace.fail_closed === true) { return { childEvidence: { blocked: true, severity_counts: emptySeverityCounts() }, trace: { ...childReviewArtifact.trace, - contract: COMPOUND_ENGINEERING_EVIDENCE_KEY, + contract: "declared_child_review_artifact", loaded: false, errors: ["child review artifact was missing or unreadable"], }, @@ -945,15 +727,32 @@ async function loadStructuredChildEvidence(childOutput: Record) }; } + const nativeReviewRound = extractNativeReviewRoundFromBody(childReviewArtifact.body, runner); + if (nativeReviewRound !== undefined) { + return { + childEvidence: nativeReviewRound.childEvidence, + trace: { + ...childReviewArtifact.trace, + ...nativeReviewRound.trace, + }, + missing: nativeReviewRound.missing, + errors: nativeReviewRound.errors, + childReviewArtifact, + }; + } + const artifactEvidence = extractJsonEvidenceFromBody(childReviewArtifact.body); - if (artifactEvidence === undefined) { - const missingError = `missing named ${COMPOUND_ENGINEERING_EVIDENCE_KEY} block in child review artifact`; + if (artifactEvidence.evidence === undefined) { + const missingError = artifactEvidence.legacyEvidenceFound + ? "legacy compound_engineering_evidence was ignored; child review artifact must use declared review_report/review_report_path content with review_evidence" + : "missing review_evidence object in child review artifact"; return { childEvidence: { severity_counts: emptySeverityCounts() }, trace: { ...childReviewArtifact.trace, - contract: COMPOUND_ENGINEERING_EVIDENCE_KEY, + contract: "declared_child_review_artifact", loaded: false, + legacy_compound_engineering_evidence_ignored: artifactEvidence.legacyEvidenceFound, errors: [missingError], }, missing: [...REVIEW_CRITERIA], @@ -962,13 +761,14 @@ async function loadStructuredChildEvidence(childOutput: Record) }; } - const converted = structuredEvidenceToReviewEvidence(artifactEvidence); + const converted = structuredEvidenceToReviewEvidence(artifactEvidence.evidence); return { childEvidence: converted.evidence, trace: { ...childReviewArtifact.trace, - contract: COMPOUND_ENGINEERING_EVIDENCE_KEY, + contract: "declared_child_review_artifact", loaded: true, + legacy_compound_engineering_evidence_ignored: artifactEvidence.legacyEvidenceFound, missing: converted.missing, errors: converted.errors, }, @@ -1036,6 +836,83 @@ async function writeCompactFinalReport(options: { ].join("\n"))); } +const reviewEvidenceCriterionSchema = Type.Object({ + satisfied: Type.Boolean(), + evidence: Type.Optional(Type.String()), + source: Type.Optional(Type.String()), + commands: Type.Optional(Type.Array(Type.Object({ + command: Type.Optional(Type.String()), + exit_code: Type.Optional(Type.Integer()), + summary: Type.Optional(Type.String()), + }, { additionalProperties: false }))), +}, { additionalProperties: false }); + +const severityCountsSchema = Type.Object({ + p0: Type.Integer(), + p1: Type.Integer(), + p2: Type.Integer(), + p3: Type.Integer(), +}, { additionalProperties: false }); + +const normalizedReviewEvidenceSchema = Type.Object({ + independent: reviewEvidenceCriterionSchema, + acceptance_mapped: reviewEvidenceCriterionSchema, + diff_aware: reviewEvidenceCriterionSchema, + validation_backed: reviewEvidenceCriterionSchema, + risk_aware: reviewEvidenceCriterionSchema, + fresh: reviewEvidenceCriterionSchema, + severity_counts: severityCountsSchema, + blocked: Type.Optional(Type.Boolean()), + conflicted: Type.Optional(Type.Boolean()), + validation_failed: Type.Optional(Type.Boolean()), +}, { additionalProperties: false, description: "Structured review evidence read from declared child review_report/review_report_path artifacts." }); + +const implementationLeafSchema = Type.Union([ + Type.String(), + Type.Number(), + Type.Boolean(), + Type.Null(), + Type.Array(Type.Union([Type.String(), Type.Number(), Type.Boolean(), Type.Null(), Type.Object({}, { additionalProperties: true })])), + Type.Object({}, { additionalProperties: true }), +]); + +const implementationSchema = Type.Union([ + Type.Object({ + kind: Type.Literal("guarded_handoff"), + requested_runner: Type.String(), + resolved_runner: resolvedRunnerSchema, + workflow: resolvedRunnerSchema, + inputs: Type.Object({}, { additionalProperties: implementationLeafSchema }), + safe_note: Type.String(), + child_workflow_launched: Type.Boolean(), + }, { additionalProperties: false }), + Type.Object({ + kind: Type.Literal("child_workflow_receipt"), + requested_runner: Type.String(), + resolved_runner: resolvedRunnerSchema, + workflow: resolvedRunnerSchema, + inputs: Type.Object({}, { additionalProperties: implementationLeafSchema }), + safe_note: Type.String(), + child_workflow_launched: Type.Boolean(), + child_exited: Type.Optional(Type.Boolean()), + outputs: Type.Object({}, { additionalProperties: implementationLeafSchema }), + evidence: Type.Optional(Type.Object({}, { additionalProperties: implementationLeafSchema })), + gate_child_run_completion: Type.Object({}, { additionalProperties: implementationLeafSchema }), + gate_review_evidence: Type.Optional(Type.Object({}, { additionalProperties: implementationLeafSchema })), + }, { additionalProperties: false }), + Type.Object({ + kind: Type.Literal("worktree_root_resolution_failed"), + requested_runner: Type.String(), + resolved_runner: resolvedRunnerSchema, + workflow: resolvedRunnerSchema, + requested_git_worktree_dir: Type.String(), + safe_note: Type.String(), + child_workflow_launched: Type.Boolean(), + }, { additionalProperties: false }), +], { description: "Typed implementation receipt union; dynamic child details are constrained to leaf objects." }); + +void normalizedReviewEvidenceSchema; + const compoundEngineeringWorkflow = defineWorkflow("compound-engineering") .description("Safe Compound Engineering loop: classify intake, scout memory, brainstorm/plan, require approval, run explicit implementation runners, gate review evidence, and optionally capture learning.") .input("prompt", Type.String({ description: "Idea, spec/plan path, work request, review target, or learning-capture request." })) @@ -1053,7 +930,7 @@ const compoundEngineeringWorkflow = defineWorkflow("compound-engineering") Type.Literal("ralph"), Type.Literal("handoff-only"), ], { default: "auto", description: "auto, goal, ralph, or handoff-only. Iteration 3 auto resolves to handoff-only; explicit goal/ralph run after approval." })) - .input("max_loops", Type.Number({ default: DEFAULT_MAX_LOOPS, description: "Maximum implementation/fix/review loops to include in handoff metadata." })) + .input("max_loops", Type.Integer({ default: DEFAULT_MAX_LOOPS, description: "Maximum implementation/fix/review loops to include in handoff metadata." })) .input("base_branch", Type.String({ default: DEFAULT_BASE_BRANCH, description: "Base branch for implementation/review handoff." })) .input("git_worktree_dir", Type.String({ default: "", description: "Optional reusable worktree directory for implementation handoff." })) .input("create_pr", Type.Boolean({ default: false, description: "Strict true authorizes PR creation in handoff inputs; default false." })) @@ -1082,7 +959,7 @@ const compoundEngineeringWorkflow = defineWorkflow("compound-engineering") .output("plan_path", Type.Optional(Type.String({ description: "Saved plan path." }))) .output("spec_path", Type.Optional(Type.String({ description: "Saved spec path." }))) .output("approved_spec_path", Type.Optional(Type.String({ description: "Approved plan/spec path." }))) - .output("implementation", Type.Optional(Type.Object({}, { additionalProperties: true, description: "Guarded handoff metadata or child-run summary." }))) + .output("implementation", Type.Optional(implementationSchema)) .output("review_report_path", Type.Optional(Type.String({ description: "Saved normalized review/evidence report path." }))) .output("learning_doc_path", Type.Optional(Type.String({ description: "Saved docs/solutions learning artifact path." }))) .run(async (ctx) => { @@ -1099,6 +976,19 @@ const compoundEngineeringWorkflow = defineWorkflow("compound-engineering") const baseBranch = text(ctx.inputs.base_branch, DEFAULT_BASE_BRANCH); const gitWorktreeDir = text(ctx.inputs.git_worktree_dir); const createPr = normalizeCreatePr(ctx.inputs.create_pr); + + if (prompt.length === 0) { + return exitWorkflow(ctx, outputRecord({ + status: "blocked", + mode, + runner, + approved: false, + artifact_dir: "", + manifest_path: "", + message: "Blocked: prompt is required.", + })); + } + const { runId, artifactDir } = await createArtifactRun(WORKFLOW_NAME, startedAt, cwd); const artifacts = new Map(); const addArtifact = (name: string, path: string): string => { @@ -1154,13 +1044,6 @@ const compoundEngineeringWorkflow = defineWorkflow("compound-engineering") return { selectedLearningMode, learningDocPath }; }; - if (prompt.length === 0) { - const message = "Blocked: prompt is required."; - await writeFinalReport({ status: "blocked", mode, runner, prompt, message }); - await writeFinalManifest({ manifestPath, runId, startedAt, input: baseInput, finalReportPath, artifacts }); - return outputRecord({ status: "blocked", mode, runner, approved: false, artifact_dir: displayPath(artifactDir), manifest_path: displayPath(manifestPath), message }); - } - await writeArtifactMarkdown("intake", join(artifactDir, "00-intake.md"), artifactMarkdown("Compound Engineering intake", [ `Resolved mode: ${mode}`, `Resolved runner: ${runner}`, @@ -1395,7 +1278,53 @@ const compoundEngineeringWorkflow = defineWorkflow("compound-engineering") stageName: `${runner} implementation`, inputs: childInputs, }); - const childOutput = normalizedChildOutput(childResult); + if (childWorkflowExited(childResult)) { + const childOutput = normalizedChildOutput(childResult, runner); + const receiptChildOutput = compactChildOutputForReceipt(childOutput); + const exitStatus = childExitStatus(childResult); + const domainStatus = domainStatusForChildExit(exitStatus); + const exitReason = childExitReason(childResult, `${runner} child workflow exited before returning declared completion evidence.`); + const childExitGate = { + state: "child_exited", + parent_status: domainStatus, + reason: exitReason, + exited: true, + exit_status: exitStatus, + exit_reason: exitReason, + }; + const implementation = outputRecord({ + kind: "child_workflow_receipt", + requested_runner: requestedRunner, + resolved_runner: runner, + workflow: runner, + inputs: childInputs, + safe_note: handoff.safe_note, + child_workflow_launched: true, + child_exited: true, + outputs: receiptChildOutput, + gate_child_run_completion: childExitGate, + }); + const message = `Plan/spec approved and ${runner} child workflow exited (${exitStatus}) before returning declared completion evidence.`; + const output = outputRecord({ + status: domainStatus, + mode, + runner, + approved: false, + artifact_dir: displayPath(artifactDir), + manifest_path: displayPath(manifestPath), + message, + brainstorm_path: brainstormPath ? displayPath(brainstormPath) : undefined, + plan_path: displayPath(planPath), + spec_path: displayPath(specPath), + approved_spec_path: displayPath(approvedSpecPath), + implementation, + }); + await writeFinalReport({ status: domainStatus, mode, runner, prompt, message, implementation }); + await writeFinalManifest({ manifestPath, runId, startedAt, input: { ...baseInput, approval_decision: approvalDecision, revisions }, finalReportPath, artifacts }); + return exitWorkflow(ctx, output, exitStatus, exitReason); + } + + const childOutput = normalizedChildOutput(childResult, runner); const receiptChildOutput = compactChildOutputForReceipt(childOutput); const childGate = gateChildRunCompletion(childOutput, runner); let reviewReportPath: string | undefined; @@ -1436,7 +1365,7 @@ const compoundEngineeringWorkflow = defineWorkflow("compound-engineering") }); } - const structuredGate = await loadStructuredChildEvidence(childOutput); + const structuredGate = await loadStructuredChildEvidence(childOutput, runner); const childReviewArtifact = structuredGate.childReviewArtifact; if (childReviewArtifact?.reportPath !== undefined) reviewReportPath = addArtifact("child-review-report", childReviewArtifact.reportPath); const childEvidence = structuredGate.childEvidence; @@ -1513,5 +1442,4 @@ const compoundEngineeringWorkflow = defineWorkflow("compound-engineering") }) .compile(); -export { compoundEngineeringWorkflow }; export default compoundEngineeringWorkflow; diff --git a/workflows/compound-engineering/lib/review-evidence.ts b/workflows/compound-engineering/lib/review-evidence.ts new file mode 100644 index 0000000..4ba3791 --- /dev/null +++ b/workflows/compound-engineering/lib/review-evidence.ts @@ -0,0 +1,556 @@ +import { readFile } from "node:fs/promises"; +import type { ResolvedImplementationRunner, ReviewCriterion, ReviewEvidence, SeverityCounts } from "../helpers.js"; + +type SufficiencyCriterion = ReviewCriterion; +export type SeverityLevel = keyof Required; + +export const SEVERITY_LEVELS: readonly SeverityLevel[] = ["p0", "p1", "p2", "p3"]; + +type CriterionPolicy = { + positive: readonly RegExp[]; + negative: readonly RegExp[]; +}; + +const VALIDATION_NO_RUN_PATTERN = /\b(?:validation\s+commands?|tests?|test\s+commands?|commands?)\s+(?:run|ran)\s*:?\s*(?:none|no\s+commands?|zero|0|n\/a)\b|\bno\s+validation\s+(?:(?:was|were)\s+)?(?:run|ran|performed)\b|\bno\s+validation\s+commands?\s+(?:(?:was|were)\s+)?run\b|\bno\s+tests?\s+(?:(?:was|were)\s+)?run\b|\bno\s+commands?\s+(?:(?:was|were)\s+)?run\b|\bvalidation\s+(?:commands?\s+)?skipped\b|\bvalidation\b.{0,50}\bskipped\b|\bvalidation\s+not\s+(?:run|performed|backed|available)\b|\bvalidation\b.{0,50}\b(?:was|were)\s+not\s+(?:run|performed|available)\b|\btests?\b.{0,40}\bskipped\b|\btests?\s+(?:were\s+)?not\s+run\b/; +const CONTEXTUAL_VALIDATION_ERROR_PATTERN = /\b(?:reported\s+)?[1-9]\d*\s+errors\b|\berror\s+count\s*:?\s*[1-9]\d*\b|\berrors\s*:?\s*[1-9]\d*\b|\bfailed\s+with\s+errors\b/; +const VALIDATION_NEGATED_SUCCESS_PATTERN = /\b(?:not|never)\s+(?:(?:all|any|every)\s+(?:[\w-]+\s+){0,6})?(?:pass(?:ed|es|ing)?|successful|success|succeed(?:ed|s|ing)?)\b|\bunsuccessful\b/; +const VALIDATION_SUCCESS_PATTERN = /\b(?:passed|passes|passing|succeeded|successful|success|status\s*:?\s*0|code\s*:?\s*0|exit\s+(?:code|status)\s*:?\s*0|exited\s+with\s+(?:exit\s+)?(?:code|status)\s*:?\s*0|returned\s+(?:exit\s+)?(?:code|status)\s*:?\s*0)\b/; + +const REVIEW_EVIDENCE_POLICIES = { + independent: { + positive: [/\bindependent(?:ly)?\b/, /\bseparate reviewer\b/, /\bfresh[-\s]context reviewer\b/], + negative: [ + /\bnot\s+independent\b/, + /\bno\s+(?:independent|separate|fresh[-\s]context)\s+review(?:er)?\b/, + /\bwithout\s+(?:an?\s+)?(?:independent|separate|fresh[-\s]context)\s+review(?:er)?\b/, + /\bself[-\s]review\b/, + ], + }, + acceptance_mapped: { + positive: [ + /\b(?:map(?:ped)?|mapping|check(?:ed)?|verify|verified|trace(?:d)?|cover(?:ed)?|coverage)\b.{0,80}\b(?:acceptance(?:\s+criteria)?|approved\s+(?:plan|spec))\b/, + /\b(?:acceptance(?:\s+criteria)?|approved\s+(?:plan|spec))\b.{0,80}\b(?:map(?:ped)?|mapping|check(?:ed)?|verify|verified|trace(?:d)?|cover(?:ed)?|coverage)\b/, + ], + negative: [ + /\bno\s+acceptance(?:\s+criteria)?\s+(?:mapping|mapped|traceability|coverage)\b/, + /\bno\s+acceptance(?:\s+criteria)?\s+(?:(?:was|were)\s+)?(?:mapped|checked|verified|traced|covered|inspected)\b/, + /\bnot\s+acceptance[-\s]mapped\b/, + /\bacceptance\s+(?:mapping\s+)?(?:not\s+)?(?:missing|skipped|absent)\b/, + /\b(?:acceptance(?:\s+criteria|\s+mapping)?|approved\s+(?:plan|spec)|spec)\b.{0,80}\b(?:not\s+(?:mapped|provided|performed|done|checked|verified|traced|covered|inspected)|(?:was|were)\s+not\s+(?:mapped|provided|performed|done|checked|verified|traced|covered|inspected)|unmapped|unchecked|unverified|untraced|uncovered|skipped|missing|absent|unavailable)\b/, + /\b(?:did not|does not|not)\s+(?:map|mapped|check|checked|verify|verified|trace|traced|cover|covered|inspect|inspected)\b.{0,80}\b(?:acceptance|approved\s+(?:plan|spec)|spec)\b/, + /\b(?:unchecked|unmapped|unverified|untraced|uncovered)\s+(?:acceptance(?:\s+criteria)?|approved\s+)?(?:spec|plan|criteria)\b/, + ], + }, + diff_aware: { + positive: [/\bdiff\b/, /\bchanged files?\b/, /\bgit status\b/, /\bgit diff\b/], + negative: [ + /\bdiff\s+(?:was\s+|were\s+)?not\s+(?:inspected|reviewed|checked)\b/, + /\bnot\s+(?:inspect(?:ed)?|review(?:ed)?|check(?:ed)?)\b.{0,40}\bdiff\b/, + /\b(?:did not|does not)\s+(?:inspect|review|check)\b.{0,40}\bdiff\b/, + /\bdiff(?:\s+review)?\b.{0,40}\b(?:skipped|missing|absent|not\s+(?:inspected|reviewed|checked|performed)|(?:was|were)\s+not\s+(?:inspected|reviewed|checked|performed))\b/, + /\b(?:unable|could not|cannot)\s+(?:to\s+)?(?:inspect|review|check)\b.{0,60}\b(?:current\s+)?(?:git\s+)?diff\b/, + /\bcould not\s+inspect\s+changed files?\b/, + /\bdiff\s+(?:was\s+)?unavailable\b/, + /\bdiff\s+(?:could\s+not|cannot)\s+be\s+(?:inspected|reviewed|checked)\b/, + /\bdiff\s+was\s+not\s+available\b/, + /\bno\s+(?:git\s+)?diff\s+(?:was\s+)?available\b/, + /\bchanged files?\s+(?:was\s+|were\s+)?(?:unavailable|not\s+(?:inspected|reviewed|checked|available))\b/, + /\bno\s+(?:git\s+)?diff\b.{0,80}\b(?:inspected|reviewed|checked)\b/, + /\bno\s+changed files?\b.{0,80}\b(?:inspected|reviewed|checked)\b/, + ], + }, + validation_backed: { + positive: [ + /\b(?:validation|tests?|commands?|bun\s+test|cargo\s+test|npm\s+test|pnpm\s+test|yarn\s+test|test\s+command)\b.{0,80}\b(?:passed|passes|passing|succeeded|successful|success|status\s*:?\s*0|code\s*:?\s*0|exit\s+(?:code|status)\s*:?\s*0|exited\s+with\s+(?:exit\s+)?(?:code|status)\s*:?\s*0|returned\s+(?:exit\s+)?(?:code|status)\s*:?\s*0)\b/, + /\b(?:passed|passes|passing|succeeded|successful|success|status\s*:?\s*0|code\s*:?\s*0|exit\s+(?:code|status)\s*:?\s*0|exited\s+with\s+(?:exit\s+)?(?:code|status)\s*:?\s*0|returned\s+(?:exit\s+)?(?:code|status)\s*:?\s*0)\b.{0,80}\b(?:validation|tests?|commands?|bun\s+test|cargo\s+test|npm\s+test|pnpm\s+test|yarn\s+test|test\s+command)\b/, + ], + negative: [ + VALIDATION_NO_RUN_PATTERN, + /\b(?:validation|tests?|commands?|bun\s+test|cargo\s+test)\b.{0,80}\b(?:failed|failures?|failing|errored|did\s+not\s+pass|does\s+not\s+pass|not\s+passing|non[-\s]?zero|(?:exit\s+)?(?:code|status)\s*:?\s*[1-9]\d*)\b/, + /\b(?:failed|failures?|failing|errored|did\s+not\s+pass|does\s+not\s+pass|not\s+passing|non[-\s]?zero|(?:exit\s+)?(?:code|status)\s*:?\s*[1-9]\d*)\b.{0,80}\b(?:validation|tests?|commands?|bun\s+test|cargo\s+test)\b/, + /\b(?:validation|tests?|commands?|bun\s+test|cargo\s+test)\b.{0,80}\b(?:reported\s+)?[1-9]\d*\s+errors\b/, + /\b(?:reported\s+)?[1-9]\d*\s+errors\b.{0,80}\b(?:validation|tests?|commands?|bun\s+test|cargo\s+test)\b/, + /\b(?:validation|tests?|commands?|bun\s+test|cargo\s+test)\b.{0,80}\b(?:error\s+count\s*:?\s*[1-9]\d*|errors\s*:?\s*[1-9]\d*|failed\s+with\s+errors)\b/, + /\b(?:error\s+count\s*:?\s*[1-9]\d*|errors\s*:?\s*[1-9]\d*|failed\s+with\s+errors)\b.{0,80}\b(?:validation|tests?|commands?|bun\s+test|cargo\s+test)\b/, + ], + }, + risk_aware: { + positive: [/\brisk\b/, /\bresidual\b/, /\bsecurity\b/], + negative: [ + /\bno\s+residual\s+risk\s+(?:assessment|review|analysis)\b/, + /\bno\s+risk\s+(?:assessment|review|analysis)\b/, + /\brisk\s+(?:assessment|review|analysis)\s+(?:not\s+)?(?:missing|skipped|absent)\b/, + /\brisk\b.{0,50}\bunknown\b/, + /\brisk\b.{0,50}\b(?:not\s+(?:assessed|reviewed|evaluated)|(?:was|were)\s+not\s+(?:assessed|reviewed|evaluated|performed))\b/, + /\bnot\s+(?:risk|residual\s+risk)[-\s]aware\b/, + ], + }, + fresh: { + positive: [ + /\b(?:review(?:ed)?|fresh|evidence)\b.{0,60}\bafter\s+(?:the\s+)?(?:final\s+diff|latest\s+change|current\s+diff)\b/, + /\bafter\s+(?:the\s+)?(?:final\s+diff|latest\s+change|current\s+diff)\b.{0,60}\b(?:review(?:ed)?|fresh|evidence)\b/, + ], + negative: [ + /\bnot\s+fresh\b/, + /\bstale\b/, + /\b(?:review(?:ed)?|evidence)\b.{0,60}\b(?:predates|before)\b.{0,40}\b(?:final\s+diff|latest\s+change|current\s+diff)\b/, + /\b(?:predates|before)\s+(?:the\s+)?(?:final\s+diff|latest\s+change|current\s+diff)\b/, + /\bnot\s+(?:after\s+)?(?:the\s+)?latest\s+change\b/, + /\bcurrent diff\b.{0,60}\bnot\s+(?:inspected|reviewed|checked)\b/, + ], + }, +} as const satisfies Record; + +function reviewSegments(value: string): string[] { + return value + .toLowerCase() + .split(/[\r\n.;]+/) + .map((segment) => segment.trim()) + .filter((segment) => segment.length > 0); +} + +function matchesAnyPattern(value: string, patterns: readonly RegExp[]): boolean { + return patterns.some((pattern) => pattern.test(value)); +} + +function criterionEvidenceState(segments: readonly string[], policy: CriterionPolicy): boolean | undefined { + if (segments.some((segment) => matchesAnyPattern(segment, policy.negative))) return false; + if (segments.some((segment) => matchesAnyPattern(segment, policy.positive))) return true; + return undefined; +} + +const VALIDATION_SUBJECT_PATTERN = /\b(?:validation|tests?|commands?|bun\s+test|cargo\s+test|npm\s+test|pnpm\s+test|yarn\s+test|test\s+command)\b/; +const VALIDATION_RUN_CONTEXT_PATTERN = /\b(?:run|ran|completed|bun\s+test|cargo\s+test|npm\s+test|pnpm\s+test|yarn\s+test|test\s+command)\b/; +const NO_VALIDATION_FAILURE_PATTERN = /\b(?:no|zero|0)\s+(?:validation\s+)?(?:commands?\s+)?(?:failed|failures?|errors?)\b|\b(?:validation\s+)?(?:failures?|errors?)\s*:\s*(?:none|no|zero|0)\b|\b(?:no|zero|0)\s+tests?\s+(?:failed|failing)\b|\bwithout\s+(?:validation\s+|test\s+)?(?:failures?|errors?)\b/g; +const VALIDATION_FAILURE_PATTERN = /\b(?:failed|failures?|failing|errored|did\s+not\s+pass|does\s+not\s+pass|not\s+passing|non[-\s]?zero|(?:exit\s+)?(?:code|status)\s*:?\s*[1-9]\d*|exited\s+with\s+(?:exit\s+)?(?:code|status)\s*:?\s*[1-9]\d*|returned\s+(?:exit\s+)?(?:code|status)\s*:?\s*[1-9]\d*)\b|\b(?:reported\s+)?[1-9]\d*\s+errors\b|\berror\s+count\s*:?\s*[1-9]\d*\b|\berrors\s*:?\s*[1-9]\d*\b|\bfailed\s+with\s+errors\b/; + +function removeNoValidationFailureAssertions(segment: string): string { + return segment.replace(NO_VALIDATION_FAILURE_PATTERN, " ").trim(); +} + +function hasValidationRunContext(segment: string): boolean { + return VALIDATION_SUBJECT_PATTERN.test(segment) && VALIDATION_RUN_CONTEXT_PATTERN.test(segment) && !VALIDATION_NO_RUN_PATTERN.test(segment); +} + +function validationConditionHasContext(segment: string, previousSegment: string): boolean { + return VALIDATION_SUBJECT_PATTERN.test(segment) || (previousSegment.length > 0 && hasValidationRunContext(previousSegment)); +} + +function hasValidationFailure(segments: readonly string[]): boolean { + for (let index = 0; index < segments.length; index += 1) { + const scrubbed = removeNoValidationFailureAssertions(segments[index]); + if (scrubbed.length === 0) continue; + + const hasFailure = VALIDATION_FAILURE_PATTERN.test(scrubbed) || CONTEXTUAL_VALIDATION_ERROR_PATTERN.test(scrubbed); + const hasNegatedSuccess = VALIDATION_NEGATED_SUCCESS_PATTERN.test(scrubbed); + if (!hasFailure && !hasNegatedSuccess) continue; + + const previous = index > 0 ? removeNoValidationFailureAssertions(segments[index - 1]) : ""; + if (validationConditionHasContext(scrubbed, previous)) return true; + } + return false; +} + +function validationEvidenceState(segments: readonly string[]): boolean | undefined { + const scrubbedSegments = segments + .map(removeNoValidationFailureAssertions) + .filter((segment) => segment.length > 0); + const validationPolicy = REVIEW_EVIDENCE_POLICIES.validation_backed; + + for (let index = 0; index < scrubbedSegments.length; index += 1) { + const segment = scrubbedSegments[index]; + if (!VALIDATION_NEGATED_SUCCESS_PATTERN.test(segment)) continue; + + const previous = index > 0 ? scrubbedSegments[index - 1] : ""; + if (validationConditionHasContext(segment, previous)) return false; + } + + if (scrubbedSegments.some((segment) => matchesAnyPattern(segment, validationPolicy.negative))) return false; + if (scrubbedSegments.some((segment) => matchesAnyPattern(segment, validationPolicy.positive))) return true; + + for (let index = 1; index < scrubbedSegments.length; index += 1) { + if (VALIDATION_SUCCESS_PATTERN.test(scrubbedSegments[index]) && hasValidationRunContext(scrubbedSegments[index - 1])) return true; + } + return undefined; +} + +export function emptySeverityCounts(): Required { + return { p0: 0, p1: 0, p2: 0, p3: 0 }; +} + +function severityFindingLines(value: string, severity: SeverityLevel): number { + const findingPattern = new RegExp(`(?:^|\\s)(?:-\\s*)?(?:\\[${severity}\\]|${severity}\\s*(?::\\s*\\S|[-–—]\\s*\\S))`, "i"); + return value + .split(/\r?\n/) + .map((line) => line.trim()) + .filter((line) => { + if (line.length === 0) return false; + if (/^#{1,6}\s*(?:p[0-3])\b/i.test(line)) return false; + if (/^(?:-\s*)?(?:p[0-3]\s*:\s*)?(?:none(?:\s+(?:found|identified|reported))?|no\s+(?:findings?|issues?|blockers?)|n\/a|nothing)\.?$/i.test(line)) return false; + return findingPattern.test(line); + }) + .length; +} + +const BLOCKED_POSITIVE_PATTERNS = [ + /\bblocked\b/, + /\bblockers?\b/, + /\bblocking\s+issues?\b/, + /\bunable\s+to\s+review\b/, + /\bmissing\s+dependenc(?:y|ies)\b/, +] as const; + +const BLOCKED_NEGATIVE_PATTERNS = [ + /\bnot\s+blocked\b/, + /\bno\s+blockers?\b/, + /\bno\s+blocking\s+issues?\b/, + /\bnot\s+unable\s+to\s+review\b/, +] as const; + +const CONFLICTED_POSITIVE_PATTERNS = [ + /\bconflict(?:ed|ing)?\s+evidence\b/, + /\bevidence\s+conflicts?\b/, + /\bconflicts?\b/, + /\bconflicted\b/, +] as const; + +const CONFLICTED_NEGATIVE_PATTERNS = [ + /\bno\s+conflicting\s+evidence\b/, + /\bno\s+conflicts?\b/, + /\bnot\s+conflicted\b/, + /\bno\s+evidence\s+conflicts?\b/, +] as const; + +function hardStopEvidenceState(segments: readonly string[], positive: readonly RegExp[], negative: readonly RegExp[]): boolean | undefined { + let found = false; + for (const segment of segments) { + for (const clause of segment.split(/,|\bbut\b/).map((part) => part.trim()).filter((part) => part.length > 0)) { + if (matchesAnyPattern(clause, negative)) continue; + if (matchesAnyPattern(clause, positive)) found = true; + } + } + return found ? true : undefined; +} + +export function parseEvidenceFromText(value: string): ReviewEvidence { + const segments = reviewSegments(value); + const validationFailed = hasValidationFailure(segments); + const severity_counts = emptySeverityCounts(); + for (const severity of SEVERITY_LEVELS) { + severity_counts[severity] = severityFindingLines(value, severity); + } + + return { + independent: criterionEvidenceState(segments, REVIEW_EVIDENCE_POLICIES.independent), + acceptance_mapped: criterionEvidenceState(segments, REVIEW_EVIDENCE_POLICIES.acceptance_mapped), + diff_aware: criterionEvidenceState(segments, REVIEW_EVIDENCE_POLICIES.diff_aware), + validation_backed: validationFailed ? false : validationEvidenceState(segments), + risk_aware: criterionEvidenceState(segments, REVIEW_EVIDENCE_POLICIES.risk_aware), + fresh: criterionEvidenceState(segments, REVIEW_EVIDENCE_POLICIES.fresh), + severity_counts, + conflicted: hardStopEvidenceState(segments, CONFLICTED_POSITIVE_PATTERNS, CONFLICTED_NEGATIVE_PATTERNS), + validation_failed: validationFailed, + blocked: hardStopEvidenceState(segments, BLOCKED_POSITIVE_PATTERNS, BLOCKED_NEGATIVE_PATTERNS), + }; +} + +export type ChildRunGate = + | { state: "approved"; parent_status?: undefined; reason: string; approved: true; status?: string } + | { state: "non_approved"; parent_status: "needs_human"; reason: string; approved?: boolean; status?: string } + | { state: "blocked"; parent_status: "blocked"; reason: string; approved?: boolean; status: string } + | { state: "missing_approval"; parent_status: "needs_human"; reason: string; approved?: boolean; status?: string }; + +export type ChildReviewArtifact = { + body: string; + evidenceText: string; + trace: Record; + reportPath?: string; +}; + +function optionalBoolean(value: unknown): boolean | undefined { + return typeof value === "boolean" ? value : undefined; +} + +export function optionalNonEmptyString(value: unknown): string | undefined { + const stringValue = typeof value === "string" ? value.trim() : ""; + return stringValue.length > 0 ? stringValue : undefined; +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +const CHILD_FAILURE_STATUSES = new Set(["blocked", "failed", "failure", "error", "errored"]); +const CHILD_NEEDS_HUMAN_STATUSES = new Set(["needs_human", "rejected", "stopped", "active", "pending", "running"]); +const CHILD_SUCCESS_STATUSES = new Set(["complete", "completed", "success", "succeeded", "passed"]); + +function childStatusIs(status: string | undefined, statuses: ReadonlySet): boolean { + return status !== undefined && statuses.has(status); +} + +export function gateChildRunCompletion(childOutput: Record, runner: Exclude): ChildRunGate { + const approved = optionalBoolean(childOutput.approved); + const status = optionalNonEmptyString(childOutput.status); + const normalizedStatus = status?.toLowerCase(); + + if (childStatusIs(normalizedStatus, CHILD_FAILURE_STATUSES)) { + return { state: "blocked", parent_status: "blocked", reason: `Child ${runner} returned status=${status}.`, approved, status }; + } + if (approved === false) { + return { state: "non_approved", parent_status: "needs_human", reason: `Child ${runner} returned approved=false.`, approved, status }; + } + if (childStatusIs(normalizedStatus, CHILD_NEEDS_HUMAN_STATUSES)) { + return { state: "non_approved", parent_status: "needs_human", reason: `Child ${runner} returned status=${status}.`, approved, status }; + } + if (approved !== true) { + return { state: "missing_approval", parent_status: "needs_human", reason: `Child ${runner} did not return approved=true.`, approved, status }; + } + if (runner === "goal") { + if (normalizedStatus === undefined) { + return { state: "missing_approval", parent_status: "needs_human", reason: "Child goal did not return status=complete.", approved, status }; + } + if (normalizedStatus !== "complete") { + return { state: "non_approved", parent_status: "needs_human", reason: `Child goal returned status=${status}; expected status=complete.`, approved, status }; + } + } else if (normalizedStatus !== undefined && !childStatusIs(normalizedStatus, CHILD_SUCCESS_STATUSES)) { + return { state: "non_approved", parent_status: "needs_human", reason: `Child ${runner} returned unknown status=${status}.`, approved, status }; + } + + return { state: "approved", reason: `Child ${runner} returned approved=true${runner === "goal" ? " and status=complete" : ""}.`, approved, status }; +} + +type CompactReviewPointer = { + kind: "output_saved_to" | "latest_review_round_artifact"; + path?: string; +}; + +const ATOMIC_FILE_ONLY_SUFFIX_PATTERN = /\s+\(\d+(?:\.\d+)?\s*(?:B|KB|MB|GB),\s*\d+\s+lines?\)\.?(?:\s+Read this file if needed\.?)?$/i; + +function unquoteMatchingPath(value: string): string { + const quote = value[0]; + if ((quote === '"' || quote === "'") && value[value.length - 1] === quote) return value.slice(1, -1); + return value; +} + +function compactReviewPointerKind(label: string): CompactReviewPointer["kind"] { + return /^latest review round artifact$/i.test(label) ? "latest_review_round_artifact" : "output_saved_to"; +} + +function parseCompactReviewPointer(value: string): CompactReviewPointer | undefined { + const firstSubstantiveLine = value + .split(/\r?\n/) + .map((line) => line.trim()) + .find((line) => line.length > 0) ?? ""; + + const match = firstSubstantiveLine.match(/^(output saved to|saved output to|saved to|latest review round artifact):\s*(.*)$/i); + if (!match) return undefined; + + const kind = compactReviewPointerKind(match[1]); + const target = match[2].trim(); + if (target.length === 0) return { kind }; + + const path = unquoteMatchingPath(target.replace(ATOMIC_FILE_ONLY_SUFFIX_PATTERN, "").trim()); + return { kind, path }; +} + +function appendMultilineReviewArtifactField(lines: string[], label: string, value: string): void { + lines.push(`${label}:`); + lines.push(...value + .replace(/\.\s+(?=P[0-3]\s*:)/gi, ".\n") + .split(/\r?\n/) + .map((line) => line.trim()) + .filter((line) => line.length > 0)); +} + +function appendReviewArtifactField(lines: string[], label: string, value: unknown): void { + if (typeof value === "number" || typeof value === "boolean") { + lines.push(`${label}: ${String(value)}`); + return; + } + if (typeof value !== "string") return; + + const trimmed = value.trim(); + if (trimmed.length === 0) return; + + if (label === "raw_text" || label === "overall_explanation") { + appendMultilineReviewArtifactField(lines, label, trimmed); + } else { + lines.push(`${label}: ${trimmed}`); + } + + if (label === "overall_correctness" && /\b(?:incorrect|wrong|not\s+correct|failed|failure)\b/i.test(trimmed)) { + lines.push(`P1: overall_correctness ${trimmed}`); + } +} + +function prioritySeverity(value: unknown): SeverityLevel | undefined { + if (typeof value === "number" && Number.isInteger(value) && value >= 0 && value <= 3) return `p${value}` as SeverityLevel; + const match = typeof value === "string" ? value.match(/\bp([0-3])\b|^\s*([0-3])\s*$/i) : undefined; + const digit = match?.[1] ?? match?.[2]; + return digit ? `p${digit}` as SeverityLevel : undefined; +} + +function appendReviewFinding(lines: string[], finding: Record): void { + const title = optionalNonEmptyString(finding.title); + const body = optionalNonEmptyString(finding.body) ?? optionalNonEmptyString(finding.description); + const priority = prioritySeverity(finding.priority ?? finding.severity); + + if (priority !== undefined) lines.push(`${priority.toUpperCase()}: ${title ?? body ?? "structured finding"}`); + if (title !== undefined) lines.push(`finding_title: ${title}`); + if (body !== undefined) lines.push(`finding_body: ${body}`); + appendReviewArtifactField(lines, "priority", finding.priority ?? finding.severity); + + const codeLocation = finding.code_location; + if (typeof codeLocation === "object" && codeLocation !== null) { + const path = optionalNonEmptyString((codeLocation as Record).absolute_file_path); + if (path !== undefined) lines.push(`file: ${path}`); + } +} + +function appendSeverityCounts(lines: string[], value: Record): void { + for (const severity of SEVERITY_LEVELS) { + const count = value[severity] ?? value[severity.toUpperCase()]; + if (typeof count === "number" && Number.isInteger(count) && count > 0) { + lines.push(`${severity.toUpperCase()}: severity count reported ${count}`); + } else if (typeof count === "number" && count === 0) { + lines.push(`${severity.toUpperCase()}: none`); + } else if (count !== undefined) { + lines.push(`P1: malformed ${severity.toUpperCase()} severity count; expected a non-negative integer`); + } + } +} + +function appendReviewArtifactJson(lines: string[], value: unknown, key = "root"): void { + if (Array.isArray(value)) { + for (const item of value) appendReviewArtifactJson(lines, item, key); + return; + } + if (typeof value !== "object" || value === null) return; + + const record = value as Record; + if (key === "findings") { + appendReviewFinding(lines, record); + } + + for (const [childKey, childValue] of Object.entries(record)) { + if (childKey === "findings" && Array.isArray(childValue)) { + for (const finding of childValue) { + if (typeof finding === "object" && finding !== null) appendReviewFinding(lines, finding as Record); + } + continue; + } + if (childKey === "severity_counts" && typeof childValue === "object" && childValue !== null) { + appendSeverityCounts(lines, childValue as Record); + continue; + } + if (/^(?:reviewer|overall_correctness|overall_explanation|raw_text|validation(?:_notes|_output|_summary|_results)?|commands_run|notes?)$/i.test(childKey)) { + appendReviewArtifactField(lines, childKey, childValue); + continue; + } + appendReviewArtifactJson(lines, childValue, childKey); + } +} + +export function reviewArtifactToEvidenceText(body: string): string { + try { + const parsed = JSON.parse(body) as unknown; + const lines: string[] = []; + appendReviewArtifactJson(lines, parsed); + return lines.length > 0 ? lines.join("\n") : body; + } catch { + return body; + } +} + +async function loadReviewArtifactPath(path: string): Promise { + const body = await readFile(path, "utf8"); + if (body.trim().length === 0) throw new Error(`Child review artifact is empty: ${path}`); + return body; +} + +async function loadReviewEvidenceArtifact(path: string, trace: Record): Promise { + const body = await loadReviewArtifactPath(path); + return { + body, + evidenceText: reviewArtifactToEvidenceText(body), + reportPath: path, + trace, + }; +} + +export async function loadChildReviewArtifact(childOutput: Record): Promise { + const reviewReportPath = optionalNonEmptyString(childOutput.review_report_path); + const inlineReviewReport = optionalNonEmptyString(childOutput.review_report); + const inlineCompactPointer = inlineReviewReport !== undefined ? parseCompactReviewPointer(inlineReviewReport) : undefined; + + if (reviewReportPath !== undefined) { + try { + return await loadReviewEvidenceArtifact(reviewReportPath, { source: "review_report_path", path: reviewReportPath, loaded: true }); + } catch (error) { + const reviewReportPathError = errorMessage(error); + return { + body: "", + evidenceText: `blocked unable to review missing dependency: child review_report_path unreadable: ${reviewReportPath}: ${reviewReportPathError}`, + trace: { source: "review_report_path", path: reviewReportPath, loaded: false, error: reviewReportPathError, fail_closed: true }, + }; + } + } + + if (inlineReviewReport !== undefined) { + if (inlineCompactPointer !== undefined) { + if (inlineCompactPointer.path !== undefined) { + try { + return await loadReviewEvidenceArtifact(inlineCompactPointer.path, { + source: "inline_review_report_pointer", + path: inlineCompactPointer.path, + loaded: true, + compact_pointer: true, + pointer_kind: inlineCompactPointer.kind, + }); + } catch (error) { + const pointerErrorMessage = errorMessage(error); + return { + body: "", + evidenceText: `blocked unable to review missing dependency: compact child review_report pointer unreadable: ${inlineCompactPointer.path}: ${pointerErrorMessage}`, + trace: { source: "inline_review_report_pointer", path: inlineCompactPointer.path, loaded: false, compact_pointer: true, pointer_kind: inlineCompactPointer.kind, error: pointerErrorMessage, fail_closed: true }, + }; + } + } + return { + body: "", + evidenceText: "blocked unable to review missing dependency: child review_report is only a compact file-only pointer without a readable path", + trace: { source: "inline_review_report_pointer", loaded: false, compact_pointer: true, pointer_kind: inlineCompactPointer.kind, fail_closed: true }, + }; + } + return { + body: inlineReviewReport, + evidenceText: reviewArtifactToEvidenceText(inlineReviewReport), + trace: { source: "inline_review_report", loaded: true }, + }; + } + + for (const key of ["ledger_path", "implementation_notes_path"] as const) { + const artifactPath = optionalNonEmptyString(childOutput[key]); + if (artifactPath === undefined) continue; + try { + return await loadReviewEvidenceArtifact(artifactPath, { source: key, path: artifactPath, loaded: true, declared_artifact_fallback: true }); + } catch (error) { + const artifactErrorMessage = errorMessage(error); + return { + body: "", + evidenceText: `blocked unable to review missing dependency: declared child ${key} unreadable: ${artifactPath}: ${artifactErrorMessage}`, + trace: { source: key, path: artifactPath, loaded: false, declared_artifact_fallback: true, error: artifactErrorMessage, fail_closed: true }, + }; + } + } + + return { + body: "", + evidenceText: "blocked unable to review missing dependency: child output did not include review_report_path, substantive review_report, ledger_path, or implementation_notes_path evidence", + trace: { source: "missing_child_review_report", loaded: false, fail_closed: true }, + }; +} + +export function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +}