diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml new file mode 100644 index 0000000..d34bcd4 --- /dev/null +++ b/.github/workflows/pages.yml @@ -0,0 +1,49 @@ +name: GitHub Pages demo + +on: + push: + branches: + - main + workflow_dispatch: + +concurrency: + group: pages + cancel-in-progress: false + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-24.04 + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + with: + persist-credentials: false + - name: Set up Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e + with: + node-version: "20.19.0" + - name: Build static demo + run: npm --prefix packages/orchestration run build:pages-demo + - name: Configure GitHub Pages + uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b + - name: Upload GitHub Pages artifact + uses: actions/upload-pages-artifact@7b1f4a764d45c48632c6b24a0339c27f5614fb0b + with: + path: dist/pages-demo + + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-24.04 + needs: build + permissions: + pages: write + id-token: write + steps: + - name: Deploy GitHub Pages + id: deployment + uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e diff --git a/.stylelintrc.json b/.stylelintrc.json new file mode 100644 index 0000000..8a72580 --- /dev/null +++ b/.stylelintrc.json @@ -0,0 +1,22 @@ +{ + "rules": { + "at-rule-property-required-list": null, + "declaration-no-important": null, + "declaration-property-value-disallowed-list": null, + "function-disallowed-list": null, + "function-url-scheme-allowed-list": null, + "function-url-scheme-disallowed-list": null, + "media-feature-name-allowed-list": null, + "media-feature-name-value-allowed-list": null, + "property-disallowed-list": null, + "selector-attribute-name-disallowed-list": null, + "selector-disallowed-list": null, + "selector-max-compound-selectors": null, + "selector-max-id": null, + "selector-max-specificity": null, + "selector-max-universal": null, + "selector-not-notation": "complex", + "time-min-milliseconds": null, + "unit-allowed-list": null + } +} diff --git a/DESIGN.md b/DESIGN.md new file mode 100644 index 0000000..316e6b6 --- /dev/null +++ b/DESIGN.md @@ -0,0 +1,15 @@ +# Visual system register + +The operator console uses the existing RAE evidence-dossier system: square geometry, restrained blue and teal evidence accents, neutral paper-like surfaces, compact system typography, tabular identifiers, thin rules, and explicit text labels beside status color. + +The workflow editor extends that system as a visual systems map: + +- a registry rail for workflow identity, revisions, and activation state +- a spatial SVG projection for topology and execution relationships +- a synchronized structured node and edge list for complete non-canvas operation +- a property inspector using native form controls +- a validation ledger, revision diff, budget display, and activation history + +Canvas dragging is optional and never the only authoring path. Standard controls retain native keyboard behavior. Focus indicators remain visible, status is communicated with words and symbols as well as color, and motion is removed when the user requests reduced motion. Responsive layouts preserve the registry, editor, and validation order at narrow widths. + +This register describes the implemented design intent. It makes no formal accessibility-conformance claim. diff --git a/PRODUCT.md b/PRODUCT.md new file mode 100644 index 0000000..5ac9ef6 --- /dev/null +++ b/PRODUCT.md @@ -0,0 +1,18 @@ +# Product register + +RAE is a source-distributed toolkit for bounded repository change, repair, and evaluation. Its supported autonomous provider is Codex. It operates on a committed Git repository in an isolated worktree by default and does not commit, push, publish, deploy, or promote workflow revisions. + +The graph-native runtime treats a versioned workflow as executable policy. Requirements, design criticism, planning, alignment, mutation, verification, and repair are arbitrary typed nodes rather than a fixed phase list. Each run owns an immutable workflow snapshot, node guidance, payload contracts, attempt envelopes, trace order, and evidence references. Existing v1 run requests remain linear when resumed. + +The operator is a loopback-only, bearer-authenticated local console. It is used to inspect runs, resolve checkpoints, stop or resume owned work, and author future workflow revisions. Active runs cannot be edited. Workflow activation is an attributed human decision that affects future runs only. + +The improvement campaign is evaluator-owned. It compares bounded workflow policy and topology candidates against frozen development and held-out matrices, records append-only lineage, and emits a recommendation. It never activates a candidate. + +Primary users are maintainers who need inspectable repository automation, explicit ownership, conservative mutation, reproducible evidence, and human control over publication and policy changes. + +Product boundaries: + +- Runtime code, judges, fixtures, safety invariants, provider selection, model selection, tools, commands, and promotion rules are not candidate-editable. +- Context graph memory is separately opt-in with `--graph-memory` and cannot authorize mutation. +- The supported execution sandbox is the repository's documented macOS evaluator backend. Unsupported hosts fail closed where that boundary applies. +- Local evidence is not publication or release evidence. diff --git a/README.md b/README.md index 70fb5bc..3f4dc3f 100644 --- a/README.md +++ b/README.md @@ -21,15 +21,19 @@ The repository is an alpha candidate. It does not publish a package, container, hosted service, or stable API. See [Release Status](RELEASE_STATUS.md) for the current release evidence. +Explore the [static Evidence Dossier demo](https://sebastianspicker.github.io/rae/). +It uses sanitized fixture data, runs no command, and stores no state. + ## Capabilities and limitations RAE currently provides: -- a ten-stage orchestration runtime with typed artifacts and pass/fail gates +- a graph-native orchestration runtime with typed nodes, joins, bounded repair loops, and immutable evidence envelopes - isolated Git worktrees for autonomous repository changes - a loopback-only operator console for run status, checkpoints, resume, and stop - Ralph audit, linting, and story-scoped fixing modes - benchmark validation, execution, comparison, calibration, and release gates +- opt-in local repository, workflow, evidence, and temporal-memory graph projections - a transactional Git co-author trailer cleaner - sanitized environment-profile templates and installers @@ -97,6 +101,8 @@ The umbrella command forwards arguments to the runtime that owns them: Use `--policy ` to select a validated orchestration policy. Use `--checkpoint-policy before-mutation` or `before-mutation-and-ship` to require operator approval at those boundaries. +Use `--graph-memory read` or `read-write` only when local graph retrieval is +required. Graph memory is `off` by default. Ralph accepts command flags and `RALPH_*` environment variables documented in its package README. @@ -127,6 +133,22 @@ Run a task in an isolated worktree: --task "Add a tested health endpoint and document its behavior" ``` +New runs use the committed graph-native workflow by default. Select a validated +workflow with `--workflow`, stop at a node with `--through`, or temporarily +start the v1 engine with `--legacy-linear`. Existing v1 requests always resume +through the linear engine. + +Inspect future workflow revisions with `graph workflow list|validate|show|diff` +and activate a reviewed revision with `graph workflow activate`. Activation +requires an attributed rationale and exact digest confirmation, and affects +future runs only. + +Workflow schema 2.1 supports bounded data-driven fan-out, item streams, +deterministic transforms, threshold joins, until-dry discovery, and logical +execution tiers. Use `--execution-profile ` to supply the operator-owned +Codex model mapping. Use `graph workflow propose` for a read-only, draft-only +Codex topology proposal; it never activates or executes the candidate. + Use `--through plan` to stop before repository mutation. The default worktree is stored under the target repository's Git metadata at `.git/rae-worktrees/`. The final output identifies the worktree and @@ -167,6 +189,18 @@ Run one benchmark split: --output-dir evals/results/local-dev ``` +Evaluate a sealed workflow-improvement campaign without activation: + +```bash +./scripts/rae.sh eval improve \ + --campaign evals/campaigns/autonomous-policy-improvement.v2.json \ + --baseline-evaluation evals/results/local/baseline-development.json \ + --candidate-policy evals/results/local/candidate-policy.json \ + --candidate-evaluation evals/results/local/candidate-development.json \ + --sealed-evaluation evals/results/local/candidate-held-out.json \ + --output-dir evals/results/local/improvement-campaign +``` + Local outputs under `evals/results/local*`, `.pipeline/`, and package runtime directories are intentionally ignored. diff --git a/TESTING.md b/TESTING.md index f2a04b3..6716de1 100644 --- a/TESTING.md +++ b/TESTING.md @@ -29,7 +29,7 @@ The root gate runs: ## Test inventory -The current tree contains 128 executable test source files and 11 referenced +The current tree contains 129 executable test source files and 11 referenced runner, helper, fixture, or configuration files. | Classification | Paths | Count | Runner or owner | @@ -42,7 +42,7 @@ runner, helper, fixture, or configuration files. | Active | `packages/loops/ralph/tests/ralph_*_test.sh` | 63 | `packages/loops/ralph/scripts/run_tests.sh` | | Active support | Ralph test runner and `tests/lib/test_helpers.sh` | 2 | Ralph shell suite | | Active | `packages/orchestration/operator/tests/*.test.mjs` | 5 | Node test runner | -| Active | `packages/orchestration/scripts/pipeline/tests/*.test.mjs` | 25 | Vitest | +| Active | `packages/orchestration/scripts/pipeline/tests/*.test.mjs` | 26 | Vitest | | Active support | Pipeline Vitest config, test helper, and two fixture modules | 4 | Pipeline Vitest suite | | Active | `packages/orchestration/skills/dev-tools/*/tests/unit/*.test.ts` | 17 | Package-local Vitest commands | | Active support | `trace-test-helpers.ts` | 1 | Trace collector tests | @@ -80,8 +80,12 @@ Orchestration: npm --prefix packages/orchestration run test:operator npm --prefix packages/orchestration run test:runner npm --prefix packages/orchestration run verify +npm --prefix packages/orchestration run benchmark:workflow-topology ``` +The topology benchmark is a deterministic scheduler fixture for event order, +critical path, and barrier idle time. It does not measure model quality. + Ralph: ```bash diff --git a/docs/assets/diagrams/diamond-versus-pipeline.svg b/docs/assets/diagrams/diamond-versus-pipeline.svg new file mode 100644 index 0000000..4512d16 --- /dev/null +++ b/docs/assets/diagrams/diamond-versus-pipeline.svg @@ -0,0 +1,15 @@ + + RAE diamond barrier compared with a streaming pipeline + A diamond waits for all parallel reviewers before reduction, while a streaming pipeline advances each mapped item independently. + + Barrier diamondStreaming pipeline + + InputLens ALens BReduce + + all join forms a declared barrier + + Map A1Map B1Map A2Map B2Map A3Map B3 + + each item advances when its predecessor settles + + diff --git a/docs/assets/diagrams/graph-edge-classification.svg b/docs/assets/diagrams/graph-edge-classification.svg new file mode 100644 index 0000000..bc9390d --- /dev/null +++ b/docs/assets/diagrams/graph-edge-classification.svg @@ -0,0 +1,18 @@ + + RAE graph edge classification + Four labeled edge types connect contract-bound workflow nodes: artifact data, item stream, condition, and loop-back. + + RAE edge contracts + + Typed artifact + artifact + Consumer + Map item + stream + Mapped stage + Decision + condition + Route + bounded loop-back + + diff --git a/docs/assets/diagrams/human-activated-workflow-lifecycle.svg b/docs/assets/diagrams/human-activated-workflow-lifecycle.svg new file mode 100644 index 0000000..1e962cb --- /dev/null +++ b/docs/assets/diagrams/human-activated-workflow-lifecycle.svg @@ -0,0 +1,13 @@ + + RAE human-activated workflow lifecycle + Codex proposes a read-only draft, RAE validates and stores it, a human confirms the digest and activates it, then a future run executes the immutable snapshot and records evidence. + + Drafts do not execute + + Codex proposesread-onlyRAE validatesand draftsHuman reviewstyped digestFuture runsnapshotsExecute andrecord evidence + + activation boundary + + A proposal never activates itself, edits concrete model mappings, or changes an active run. + + diff --git a/docs/assets/diagrams/routing-and-quorum.svg b/docs/assets/diagrams/routing-and-quorum.svg new file mode 100644 index 0000000..176e1f6 --- /dev/null +++ b/docs/assets/diagrams/routing-and-quorum.svg @@ -0,0 +1,14 @@ + + RAE conditional routing and diverse-lens quorum + A router selects bounded specialist paths whose independent results enter a three-of-four quorum before verification. + + Route, inspect, verify + Route + + Contract lensSafety lensTesting lensScope lens + + 3 of 4quorum + + Conditional edges choose work. Quorum edges define evidence sufficiency. + + diff --git a/docs/assets/diagrams/until-dry-convergence.svg b/docs/assets/diagrams/until-dry-convergence.svg new file mode 100644 index 0000000..5a9f381 --- /dev/null +++ b/docs/assets/diagrams/until-dry-convergence.svg @@ -0,0 +1,12 @@ + + RAE until-dry discovery convergence + Discovery rounds admit only unseen stable keys, record accepted and rejected keys in the seen set, and stop when a round yields no new item. + + Until-dry means globally unseen + + Round 1: A, B, Cfresh: A, B, CRound 2: B, Dfresh: DRound 3: A, Dfresh: none + + seen keys after every decision: A, B, C, Dno fresh key means converged + Rejected findings remain seen, so the loop cannot rediscover them indefinitely. + + diff --git a/docs/assets/screenshots/rae-agent-safety.svg b/docs/assets/screenshots/rae-agent-safety.svg index b310569..b38569f 100644 --- a/docs/assets/screenshots/rae-agent-safety.svg +++ b/docs/assets/screenshots/rae-agent-safety.svg @@ -1,11 +1,11 @@ - + RAE autonomous agent safety defaults Deterministic terminal capture generated from the live RAE CLI. - + @@ -34,23 +34,29 @@ --provider <name> auto or codex (command is test-integration only) --model <id> Optional Codex model override --reasoning-effort <level> low, medium, high, or xhigh - --timeout-seconds <n> Per-phase timeout (default: 1800) - --policy <path> Validated data-only autonomous policy JSON - --checkpoint-policy <mode> Human pause mode: none, before-mutation, or before-mutation-and-ship - --in-place Modify a clean target checkout directly - --through <phase> Stop after one named phase (default: release-readiness) - --run-id <id> Resume an existing run (resume command only) - --json Emit the final result as JSON - - Custom command-provider options: - --agent-command <path> Executable implementing the rae-agent-v1 stdin/stdout protocol - --agent-arg <value> Argument for the command; repeat as needed - --allow-unsafe-command-provider - Explicitly enable the unsandboxed test-integration provider + --execution-profile <file> Operator-owned logical tier to Codex mapping + --timeout-seconds <n> Per-phase timeout (default: 1800) + --policy <path> Validated data-only autonomous policy JSON + --workflow <path> Explicit graph-native workflow JSON for a new run + --legacy-linear Start a temporary v1 ten-phase run + --checkpoint-policy <mode> Human pause mode: none, before-mutation, or before-mutation-and-ship + --graph-memory <mode> Local graph mode: off, read, or read-write (default: off) + --in-place Modify a clean target checkout directly + --through <node-id> Stop after one workflow node + --max-concurrency <n> Concurrent readers, from 1 to 4 (default: 4) + --max-repair-rounds <n> Repair iterations, from 1 to 5 (default: 5) + --run-id <id> Resume an existing run (resume command only) + --json Emit the final result as JSON - Safety defaults: - - run creates an isolated Git worktree unless --in-place is explicit - - Codex phases use read-only or workspace-write sandbox modes as appropriate - - agents may not commit, push, publish, install dependencies, or use network infrastructure - - the custom command provider always fails doctor and cannot run without an unsafe opt-in + Custom command-provider options: + --agent-command <path> Executable implementing the rae-agent-v1 stdin/stdout protocol + --agent-arg <value> Argument for the command; repeat as needed + --allow-unsafe-command-provider + Explicitly enable the unsandboxed test-integration provider + + Safety defaults: + - run creates an isolated Git worktree unless --in-place is explicit + - Codex phases use read-only or workspace-write sandbox modes as appropriate + - agents may not commit, push, publish, install dependencies, or use network infrastructure + - the custom command provider always fails doctor and cannot run without an unsafe opt-in diff --git a/docs/assets/screenshots/rae-cli.svg b/docs/assets/screenshots/rae-cli.svg index 417679c..8f48953 100644 --- a/docs/assets/screenshots/rae-cli.svg +++ b/docs/assets/screenshots/rae-cli.svg @@ -1,11 +1,11 @@ - + RAE command map Deterministic terminal capture generated from the live RAE CLI. - + @@ -22,45 +22,47 @@ Run umbrella verification doctor Check runtime prerequisites and entrypoints agent <subcommand> [args] Run the autonomous coding-agent orchestrator - operator serve [args] Serve the authenticated loopback operator console - task route [args] Route a task spec and emit a planned run card - checkpoint <subcommand> [args] Create or resolve human checkpoint cards - orchestrate <subcommand> [args] Run the phased orchestration package - worktree <subcommand> [args] Run worktree-native orchestration aliases - ralph <subcommand> [args] Run Ralph or bootstrap its embedded template - hygiene <tool> [args] Run narrow maintenance tooling - eval <subcommand> [args] Run eval metadata harness commands - release-gate [args] Evaluate release-blocking benchmark gates - workflow <family> [args] Run umbrella workflow aliases - help Show this help - - Examples: - ./scripts/rae.sh doctor - ./scripts/rae.sh agent doctor - ./scripts/rae.sh agent run --task "Add a tested health endpoint and document it" - ./scripts/rae.sh operator serve --project /absolute/path/to/repository - ./scripts/rae.sh task route --task-spec evals/datasets/tool-selection/tool-selection-core.task-specs.json --task-id - tool-selection-dev-orchestration --output evals/results/planned.json - ./scripts/rae.sh orchestrate init - ./scripts/rae.sh orchestrate run-stage --run-id <id> --phase arm - ./scripts/rae.sh orchestrate record-review-state --run-id <id> --state explain --status completed - ./scripts/rae.sh orchestrate summarize-progress --run-id <id> - ./scripts/rae.sh ralph --status - ./scripts/rae.sh ralph bootstrap-template /tmp/demo-repo - ./scripts/rae.sh hygiene coauthor-cleaner --help - ./scripts/rae.sh checkpoint create --output evals/results/checkpoint.json --run-id demo --task-id task --gate-id - review --title "Review" - ./scripts/rae.sh eval validate - ./scripts/rae.sh eval run --benchmark-card evals/benchmarks/tool-selection-core.benchmark-card.json --split dev - --output-dir evals/results/tmp - ./scripts/rae.sh eval outcome --task-bundle evals/datasets/autonomous-outcomes/core.task-bundle.json --fixture-root - evals/fixtures/autonomous-outcomes --policy packages/orchestration/policies/default.autonomous-policy.json --split - dev --repeats 2 --output-dir evals/results/outcomes/dev --acknowledge-provider-usage - ./scripts/rae.sh release-gate --benchmark-card evals/benchmarks/tool-selection-core.benchmark-card.json --run-card - evals/results/tmp/run-card.json --regression-report evals/results/tmp/regression.json --ledger - evals/results/tmp/result-ledger.jsonl --output evals/results/tmp/release-gate.json - ./scripts/rae.sh worktree init . - ./scripts/rae.sh worktree summary --run-id <id> - ./scripts/rae.sh workflow repo-audit bootstrap /tmp/demo-repo - ./scripts/rae.sh workflow long-horizon init + graph <subcommand> [args] Build and query local graph projections and memory + operator serve [args] Serve the authenticated loopback operator console + task route [args] Route a task spec and emit a planned run card + checkpoint <subcommand> [args] Create or resolve human checkpoint cards + orchestrate <subcommand> [args] Run the phased orchestration package + worktree <subcommand> [args] Run worktree-native orchestration aliases + ralph <subcommand> [args] Run Ralph or bootstrap its embedded template + hygiene <tool> [args] Run narrow maintenance tooling + eval <subcommand> [args] Run eval metadata harness commands + release-gate [args] Evaluate release-blocking benchmark gates + workflow <family> [args] Run umbrella workflow aliases + help Show this help + + Examples: + ./scripts/rae.sh doctor + ./scripts/rae.sh agent doctor + ./scripts/rae.sh agent run --task "Add a tested health endpoint and document it" + ./scripts/rae.sh graph build --project-root /absolute/path/to/repository + ./scripts/rae.sh operator serve --project /absolute/path/to/repository + ./scripts/rae.sh task route --task-spec evals/datasets/tool-selection/tool-selection-core.task-specs.json --task-id + tool-selection-dev-orchestration --output evals/results/planned.json + ./scripts/rae.sh orchestrate init + ./scripts/rae.sh orchestrate run-stage --run-id <id> --phase arm + ./scripts/rae.sh orchestrate record-review-state --run-id <id> --state explain --status completed + ./scripts/rae.sh orchestrate summarize-progress --run-id <id> + ./scripts/rae.sh ralph --status + ./scripts/rae.sh ralph bootstrap-template /tmp/demo-repo + ./scripts/rae.sh hygiene coauthor-cleaner --help + ./scripts/rae.sh checkpoint create --output evals/results/checkpoint.json --run-id demo --task-id task --gate-id + review --title "Review" + ./scripts/rae.sh eval validate + ./scripts/rae.sh eval run --benchmark-card evals/benchmarks/tool-selection-core.benchmark-card.json --split dev + --output-dir evals/results/tmp + ./scripts/rae.sh eval outcome --task-bundle evals/datasets/autonomous-outcomes/core.task-bundle.json --fixture-root + evals/fixtures/autonomous-outcomes --policy packages/orchestration/policies/default.autonomous-policy.json --split + dev --repeats 2 --output-dir evals/results/outcomes/dev --acknowledge-provider-usage + ./scripts/rae.sh release-gate --benchmark-card evals/benchmarks/tool-selection-core.benchmark-card.json --run-card + evals/results/tmp/run-card.json --regression-report evals/results/tmp/regression.json --ledger + evals/results/tmp/result-ledger.jsonl --output evals/results/tmp/release-gate.json + ./scripts/rae.sh worktree init . + ./scripts/rae.sh worktree summary --run-id <id> + ./scripts/rae.sh workflow repo-audit bootstrap /tmp/demo-repo + ./scripts/rae.sh workflow long-horizon init diff --git a/docs/governance/quality-policy.md b/docs/governance/quality-policy.md index 6e20129..8fed2b7 100644 --- a/docs/governance/quality-policy.md +++ b/docs/governance/quality-policy.md @@ -1,7 +1,7 @@ --- status: stable owner: core -last_reviewed: 2026-07-10 +last_reviewed: 2026-07-31 source_of_truth: ../reference/contracts/quality-gates.md evidence_links: ../reference/invariants/determinism-contracts.md --- @@ -35,6 +35,10 @@ exclusions. ## Exact analyzer exceptions +- Bandit `B101` remains enforced for production Python. Codacy findings under + `evals/tests/` and `tests/` are classified as `TestCode` because pytest + assertions are the executable test contract; Ruff independently enforces + `S101` outside those test paths. - Bandit `B404` is omitted because it reports imports rather than executable sinks; Bandit `B603` remains enabled for every subprocess call site. - OpenGrep's Python `dangerous-subprocess-use-audit` rule is omitted because diff --git a/docs/reference/claims/bibliography.md b/docs/reference/claims/bibliography.md index b61eee9..4f9c588 100644 --- a/docs/reference/claims/bibliography.md +++ b/docs/reference/claims/bibliography.md @@ -228,6 +228,52 @@ https://doi.org/10.1518/001872095779049543 Kahneman. "Thinking, Fast and Slow." 2011. https://us.macmillan.com/books/9780374533557/thinkingfastandslow +## Graph retrieval and memory sources + +### SRC-W3C-PROV-O { #src-w3c-prov-o } + +W3C. "PROV-O: The PROV Ontology." April 30, 2013. +https://www.w3.org/TR/prov-o/ + +### SRC-GRAPHRAG-BENCH { #src-graphrag-bench } + +GraphRAG-Bench. "GraphRAG-Bench." Accessed July 29, 2026. +https://graphrag-bench.github.io/ + +### SRC-CODEXGRAPH { #src-codexgraph } + +CodexGraph. "Bridging Large Language Models and Code Repositories via Code +Graph Databases." NAACL 2025. +https://aclanthology.org/2025.naacl-long.7/ + +### SRC-REPOGRAPH { #src-repograph } + +RepoGraph. "Enhancing AI Software Engineering with Repository-level Code +Graph." 2024. +https://arxiv.org/abs/2410.14684 + +### SRC-DOES-MEMORY-NEED-GRAPHS { #src-does-memory-need-graphs } + +"Does Memory Need Graphs?" ACL 2026. +https://aclanthology.org/2026.acl-long.1232/ + +### SRC-GRAPHITI { #src-graphiti } + +"Zep: A Temporal Knowledge Graph Architecture for Agent Memory." 2025. +https://arxiv.org/abs/2501.13956 + +### SRC-GRAPHRAG-UNDER-FIRE { #src-graphrag-under-fire } + +"GraphRAG under Fire: Probing Robustness of Graph-Based Retrieval-Augmented +Generation." 2025. +https://arxiv.org/abs/2501.14050 + +### SRC-LONGMEMEVAL-V2 { #src-longmemeval-v2 } + +"LongMemEval-V2: Benchmarking Memory-Augmented Agents in Long-Horizon +Interactive Environments." 2026. +https://arxiv.org/abs/2605.12493 + ## Coverage note The bibliography is also a thesis-support surface for the documentation corpus. diff --git a/docs/reference/claims/claims-ledger.md b/docs/reference/claims/claims-ledger.md index 0df362a..c0e968b 100644 --- a/docs/reference/claims/claims-ledger.md +++ b/docs/reference/claims/claims-ledger.md @@ -46,6 +46,7 @@ evidence_links: evidence-index.md | CLM-019 | Reliability and benchmark claims require explicit threats-to-validity, contamination, and uncertainty analysis before publication-strength interpretation. | governance_rule | adopted | [Evidence Index](evidence-index.md#clm-019) | [Dossier](dossiers/clm-019-validity-doctrine.md) | | CLM-020 | Failure analysis is more diagnostic when representation, inference, coordination, and governance failures are separated instead of collapsed into one label. | engineering_heuristic | adopted | [Evidence Index](evidence-index.md#clm-020) | [Dossier](dossiers/clm-020-layered-failure-model.md) | | CLM-021 | Negative results should be preserved as first-class evidence when they constrain interpretation, calibration, or future design. | governance_rule | adopted | [Evidence Index](evidence-index.md#clm-021) | [Dossier](dossiers/clm-021-negative-results.md) | +| CLM-022 | Graph-informed repository context should remain experimental until it improves localization or reduces context under frozen held-out evaluation without reducing task passes or crossing repository and protected-path boundaries. | governance_rule | adopted | [Evidence Index](evidence-index.md#clm-022) | [Graph Contract](../contracts/graph-memory.md#experimental-status) | ## Status meanings diff --git a/docs/reference/claims/evidence-index.md b/docs/reference/claims/evidence-index.md index 22fa113..1511d8e 100644 --- a/docs/reference/claims/evidence-index.md +++ b/docs/reference/claims/evidence-index.md @@ -61,6 +61,8 @@ evidence_links: bibliography.md - Dossier: [CLM-008 coordination topology](dossiers/clm-008-coordination-topology.md) - Internal anchor: `docs/explanation/science/coordination-cost.md` - Internal anchor: `packages/orchestration/docs/ORCHESTRATION_POLICY.md` +- Internal anchor: `docs/tutorials/graph-engineering-with-codex.md` +- Internal anchor: `packages/orchestration/scripts/pipeline/lib/workflow-scheduler-v21.mjs` - External anchor: [Amdahl 1967](bibliography.md#src-amdahl-1967) - External anchor: [Conway 1968](bibliography.md#src-conway-1968) - External anchor: [Cataldo et al.](bibliography.md#src-cataldo-congruence) @@ -108,6 +110,8 @@ evidence_links: bibliography.md - Dossier: [CLM-016 cognitive tiering](dossiers/clm-016-cognitive-tiering.md) - Internal anchor: `docs/explanation/science/cognitive-tiering.md` - Internal anchor: `docs/explanation/supplementary/design-axioms.md` +- Internal anchor: `packages/orchestration/contracts/workflows/execution-profile-v1.schema.json` +- Internal anchor: `docs/tutorials/graph-engineering-with-codex.md` - External anchor: [Kahneman](bibliography.md#src-kahneman-fast-slow) - External anchor: [Bainbridge automation](bibliography.md#src-bainbridge-automation) - External anchor: [Parasuraman and Riley](bibliography.md#src-parasuraman-riley) @@ -150,6 +154,17 @@ evidence_links: bibliography.md - External anchor: [Pineau reproducibility report](bibliography.md#src-pineau-reproducibility) - External anchor: [Smaldino bad science](bibliography.md#src-smaldino-bad-science) +### CLM-022 + +- Internal anchor: `docs/reference/contracts/graph-memory.md` +- Internal anchor: `packages/orchestration/contracts/graph/` +- Internal anchor: `packages/orchestration/contracts/workflows/workflow-v2.1.schema.json` +- Regression evidence: `packages/orchestration/scripts/pipeline/tests/graph.test.mjs` +- External anchor: [GraphRAG-Bench](https://graphrag-bench.github.io/) +- External anchor: [Does Memory Need Graphs?](https://aclanthology.org/2026.acl-long.1232/) +- External anchor: [CodexGraph](https://aclanthology.org/2025.naacl-long.7/) +- External anchor: [LongMemEval-V2](https://arxiv.org/abs/2605.12493) + ### CLM-011 - Dossier: [CLM-011 explicit routing](dossiers/clm-011-explicit-routing.md) diff --git a/docs/reference/cli/umbrella.md b/docs/reference/cli/umbrella.md index 8c327ae..37569e1 100644 --- a/docs/reference/cli/umbrella.md +++ b/docs/reference/cli/umbrella.md @@ -18,6 +18,7 @@ before dispatching to the package that owns each command. | `verify` | `scripts/verify.sh` | Run repository verification | | `doctor` | `scripts/rae.sh` | Check runtime versions, tools, and entrypoints | | `agent` | orchestration autonomous CLI | Run, inspect, stop, or resume an autonomous workflow | +| `graph` | orchestration graph CLI | Build and query local projections or manage cross-run memory | | `operator serve` | orchestration operator console | Serve the loopback console for allowlisted repositories | | `task route` | evaluation router | Select a runtime for one task specification | | `checkpoint` | evaluation checkpoint CLI | Create or resolve an operator checkpoint | @@ -39,6 +40,7 @@ Subcommand options are owned by the selected runtime: ```bash ./scripts/rae.sh agent --help +./scripts/rae.sh graph --help ./scripts/rae.sh orchestrate --help ./scripts/rae.sh ralph --help ./scripts/rae.sh eval --help @@ -94,6 +96,53 @@ Resume after correcting an environmental failure: RAE does not expose commit, push, publish, or deploy actions. Supported runs reject protected Git-state changes. +Graph retrieval is disabled by default. Enable current, trusted local retrieval +for one run with `--graph-memory read`, or admit verified outcomes and +quarantine model-proposed candidates with `--graph-memory read-write`. The mode +is immutable on resume. + +Use an operator-owned execution profile when workflow nodes declare logical +tiers: + +```bash +./scripts/rae.sh agent run \ + --project-root /path/to/target-repository \ + --execution-profile /absolute/path/to/execution-profile.json \ + --task "Implement and verify the requested change" +``` + +`--execution-profile` is mutually exclusive with `--model` and +`--reasoning-effort`. The validated profile and canonical digest are stored in +the run request and remain immutable on resume. + +## Local graph and memory + +```bash +./scripts/rae.sh graph build --project-root /path/to/target-repository +./scripts/rae.sh graph status --project-root /path/to/target-repository +./scripts/rae.sh graph query --project-root /path/to/target-repository \ + --seed 'File:src/main.js' +``` + +The graph is local, rebuildable, and advisory. It cannot modify gates, +checkpoints, policies, evaluators, Git state, publication state, or plan +ownership. See the [graph and memory contract](../contracts/graph-memory.md). + +Workflow revisions use the same graph command family: + +```bash +./scripts/rae.sh graph workflow list --project-root /path/to/target-repository +./scripts/rae.sh graph workflow validate --project-root /path/to/target-repository \ + --workflow-file /absolute/path/to/workflow.json +./scripts/rae.sh graph workflow propose --project-root /path/to/target-repository \ + --task "Design a bounded topology" --base-workflow graph-native-default \ + --actor "operator-name" --rationale "Draft for review" +``` + +`propose` starts one read-only, ephemeral structured-output session and permits +one correction after local validation. It stores only a valid attributed draft. +It does not activate or execute the draft. + ## Operator console ```bash diff --git a/docs/reference/contracts/artifact-schemas.md b/docs/reference/contracts/artifact-schemas.md index e03ec37..115bdba 100644 --- a/docs/reference/contracts/artifact-schemas.md +++ b/docs/reference/contracts/artifact-schemas.md @@ -28,6 +28,16 @@ Current imported schema set includes: - quality report - release readiness - execution trace +- graph manifest, node, edge, context bundle, and memory decision +- graph-native workflow 2.0 and immutable node-result envelope 2.0 +- graph-native workflow and node-instance envelope 2.1 +- operator-owned execution profile with economy, standard, and judgment tiers + +Version 2.1 adds bounded maps, item streams, allowlisted transforms, threshold +joins, typed failure collection, until-dry convergence, and immutable instance +identity. Version 2.0 remains a separate accepted contract for existing run +snapshots and locally activated revisions. RAE does not rewrite private +registries or migrate stored runs automatically. Umbrella eval/runtime schemas additionally include: diff --git a/docs/reference/contracts/graph-memory.md b/docs/reference/contracts/graph-memory.md new file mode 100644 index 0000000..12a8f34 --- /dev/null +++ b/docs/reference/contracts/graph-memory.md @@ -0,0 +1,188 @@ +--- +status: experimental +owner: orchestration +last_reviewed: 2026-07-29 +source_of_truth: packages/orchestration/contracts/graph +evidence_links: ../claims/evidence-index.md +--- + +# Local Graph and Memory Contracts + +RAE can build four local graph projections: repository structure, workflow +state, run evidence, and cross-run memory. The feature is optional. Autonomous +runs use `--graph-memory off` unless an operator explicitly selects `read` or +`read-write`. + +The graph augments retrieval and explanations. It does not replace artifacts, +`trace.jsonl`, gates, checkpoints, policies, evaluator code, Git state, or plan +ownership. + +## Storage + +One run projection is stored under: + +```text +.pipeline/runs//graph/ + manifest.json + nodes.jsonl + edges.jsonl + contexts/.json +``` + +Repository-only builds use a stable synthetic run ID derived from the current +snapshot. Cross-run memory is stored outside the public worktree in the target +repository's Git common directory at `/rae-memory/v1/`. + +Graph and memory files use owner-only permissions, atomic replacement, and an +exclusive memory lock. Human promotion, rejection, supersession, and +invalidation records are append-only. Projections and admitted facts are +rebuildable from source artifacts. + +Repository identity is the SHA-256 digest of the canonical Git common +directory. Snapshot identity combines the `HEAD` tree digest with a digest of +the dirty overlay. Runtime files under `.pipeline/` do not affect that overlay. + +## Record model + +Node, edge, manifest, context, and memory-decision schemas live under +`packages/orchestration/contracts/graph/`. Every node and edge records its +graph family, repository and run namespace, stable logical ID, +content-addressed version ID, source reference and digest, projector version, +transaction time, validity interval, and trust class. + +Trust classes are enforced as filters: + +- `authoritative` covers repository-owned contracts, Git identity, captured + commands, gates, and human decisions +- `verified-derived` covers deterministic relations reconstructed from those + sources +- `model-proposed` covers relations extracted from model-authored artifacts +- `untrusted` covers quarantined or conflicting memory candidates + +The provenance fields are a compact JSON profile informed by +[W3C PROV-O](https://www.w3.org/TR/prov-o/). RAE does not add RDF storage or an +external graph service. + +## Repository projection + +The projector reads tracked regular files and plan-owned changed or new files. +It excludes symlinks, submodules, binaries, credential-like paths, `.pipeline` +state, files outside the canonical repository, and files larger than 1 MiB. + +Dependency-free extractors record exact literal path relationships for JSON, +TOML, JavaScript module syntax, CommonJS, shell sourcing, and Markdown links. +Python imports are parsed with the required Python runtime's standard `ast` +module. Unsupported languages retain file and exact-reference relationships. +The projector does not infer authoritative symbol, call, or data-flow edges. + +Builds fail closed for orphan edges, duplicate IDs, unresolved sources, digest +mismatches, invalid validity intervals, cross-repository records, malformed +JSONL, or configured size bounds. Completed-run projections additionally +require every MUST requirement to reach a plan task, test case, captured +command, and gate decision. + +## Retrieval + +Queries rank exact paths and identifiers first, then lexical overlap, then +bounded graph distance. Trust and current-source validity are hard filters. +Each result includes its source reference, digest, selection reason, traversal +path, score components, staleness status, and source snippet. + +Traversal depth is limited to four, output is limited to 200 records, source +files are limited to 1 MiB, and projections are limited to 250,000 nodes and +1,000,000 edges. Autonomous phase retrieval currently requests at most 50 run +records and 50 admitted memory records. A limit or validation failure stops the +opted-in graph operation. The default non-graph workflow remains available. + +## Temporal memory + +`read` retrieves only current `authoritative` and `verified-derived` facts from +the same repository identity. `read-write` also imports successful recorded +outcomes and quarantines model-proposed candidates after run completion. + +Changed facts are superseded rather than overwritten. Retrieval excludes +rejected, superseded, invalidated, stale, conflicting, and cross-repository +facts. Promotion requires a candidate ID, actor, rationale, and a safe +repository-relative corroborating source. Rejection preserves the candidate +and decision. + +Memory does not broaden plan ownership, provider access, mutation scope, or +publication authority. + +## CLI + +```bash +./scripts/rae.sh graph build --project-root /path/to/repository +./scripts/rae.sh graph status --project-root /path/to/repository +./scripts/rae.sh graph query --project-root /path/to/repository \ + --seed 'File:src/main.js' +./scripts/rae.sh graph explain --project-root /path/to/repository \ + --run-id --node 'Requirement:REQ-001' +./scripts/rae.sh graph memory list --project-root /path/to/repository +``` + +Use `--json` for the contract-defined representation. Human-readable key/value +output is the default. + +## Threat model + +The primary risks are prompt injection in source text, memory poisoning, stale +facts, high-degree hub manipulation, topology fabricated by a model, protected +path ingestion, and cross-project leakage. RAE limits these risks through exact +extractors, source digests, repository namespaces, hard trust filters, +quarantine, bounded traversal, credential-path exclusion, and source snippets. + +Graph text remains untrusted input to a provider. Operators must not treat a +relationship or summary as authorization. Raw prompts, provider metadata, +absolute paths, untrusted memory text, and unrestricted queries are excluded +from the operator API. The operator receives only health counts. + +## Experimental status + +The projection and safety contracts have deterministic local tests. Graph- +informed execution remains experimental until the frozen retrieval benchmark +shows one of these outcomes without reducing held-out task passes: + +- Recall@10 improves by at least 10 percentage points +- recall is preserved while context tokens fall by at least 25 percent + +It must also show zero cross-project or protected-path leakage, p95 query +latency no greater than 250 ms on the 100,000-node fixture, and projection time +no greater than 30 seconds on the 10,000-file fixture. No such result is +claimed by this contract page. + +The checked-in 50-task retrieval set and local comparison runner live under +`evals/datasets/graph-context/` and +`packages/orchestration/scripts/eval/graph-context-benchmark.mjs`. The runner +does not execute a provider, so task pass count remains unresolved and its +result cannot satisfy the experimental exit criteria by itself. + +The design is informed by evidence that graph retrieval is useful for +relational and repository-structure questions but is not uniformly better than +strong flat retrieval. See [GraphRAG-Bench](https://graphrag-bench.github.io/), +[CodexGraph](https://aclanthology.org/2025.naacl-long.7/), +[RepoGraph](https://arxiv.org/abs/2410.14684), and +[Does Memory Need Graphs?](https://aclanthology.org/2026.acl-long.1232/). +Temporal and security boundaries are informed by +[Graphiti](https://arxiv.org/abs/2501.13956), +[GraphRAG under Fire](https://arxiv.org/abs/2501.14050), and +[LongMemEval-V2](https://arxiv.org/abs/2605.12493). + +## Current limitations + +- Rich language-specific symbol and call graphs require a future adapter. +- Memory promotion is a local CLI operation, not an operator-console control. +- The operator exposes health counts but no unrestricted graph browser. +- Benchmark thresholds must be satisfied before graph execution can leave + experimental status. + +## Source note + +- [W3C PROV-O](../claims/bibliography.md#src-w3c-prov-o) +- [GraphRAG-Bench](../claims/bibliography.md#src-graphrag-bench) +- [CodexGraph](../claims/bibliography.md#src-codexgraph) +- [RepoGraph](../claims/bibliography.md#src-repograph) +- [Does Memory Need Graphs?](../claims/bibliography.md#src-does-memory-need-graphs) +- [Graphiti](../claims/bibliography.md#src-graphiti) +- [GraphRAG under Fire](../claims/bibliography.md#src-graphrag-under-fire) +- [LongMemEval-V2](../claims/bibliography.md#src-longmemeval-v2) diff --git a/docs/reference/terminology.md b/docs/reference/terminology.md index e7361b7..b3f721a 100644 --- a/docs/reference/terminology.md +++ b/docs/reference/terminology.md @@ -35,6 +35,15 @@ evidence_links: claims/claims-ledger.md - `orchestration` A staged workflow that separates intake, design, build, and verification into bounded phases. +- `Codex subagent` + A delegated collaborator inside one native Codex task. It is distinct from a + RAE node, which starts a fresh durable `codex exec` session. +- `node instance` + One immutable execution of a logical workflow node, optionally bound to a + stable mapped-item key. +- `execution tier` + A workflow-owned logical request for economy, standard, or judgment work. An + operator-owned execution profile resolves it to concrete Codex settings. ## Thesis validation diff --git a/docs/tutorials/autonomous-code-change.md b/docs/tutorials/autonomous-code-change.md index 6207261..ffdfa6a 100644 --- a/docs/tutorials/autonomous-code-change.md +++ b/docs/tutorials/autonomous-code-change.md @@ -38,6 +38,7 @@ deterministic tools and benchmarks can run without one. ./scripts/rae.sh agent run \ --project-root /path/to/target-repo \ --checkpoint-policy before-mutation-and-ship \ + --graph-memory off \ --task "Add a tested health endpoint and document its response contract" ``` @@ -71,6 +72,12 @@ publish, or deploy action, and supported Codex runs reject protected Git-state changes after every phase. A completed run therefore ends in `implemented-awaiting-human-release-review`. +Graph retrieval is explicitly optional. `--graph-memory read` adds bounded, +source-backed repository and admitted memory context. `read-write` also records +verified completed-run outcomes and quarantines model-proposed candidates. The +default `off` mode performs no graph read or memory write. Graph context cannot +broaden the plan's owned paths or change a gate or checkpoint. + ![Deterministic output from `rae.sh agent --help` showing the isolated-worktree default, sandbox modes, prohibited actions, and command-provider opt-in.](../assets/screenshots/rae-agent-safety.svg) diff --git a/docs/tutorials/graph-engineering-with-codex.md b/docs/tutorials/graph-engineering-with-codex.md new file mode 100644 index 0000000..580764c --- /dev/null +++ b/docs/tutorials/graph-engineering-with-codex.md @@ -0,0 +1,258 @@ +--- +status: experimental +owner: orchestration +last_reviewed: 2026-07-30 +source_of_truth: packages/orchestration/contracts/workflows/workflow-v2.1.schema.json +evidence_links: ../reference/claims/evidence-index.md +--- + +# Graph Engineering with Codex and RAE + +This course has two layers. The topology layer explains what work can proceed +and what must wait. The contract layer names the workflow fields that make that +decision durable and reviewable. + +RAE and Codex solve different coordination problems. Native Codex +[subagents](https://learn.chatgpt.com/docs/agent-configuration/subagents) are +collaborators inside one Codex task. A RAE agent node starts a fresh, +ephemeral [`codex exec`](https://learn.chatgpt.com/docs/non-interactive-mode) +session and persists its validated result as workflow evidence. RAE also uses +isolated Git [worktrees](https://learn.chatgpt.com/docs/environments/git-worktrees) +for writer runs. It does not preserve a shared model conversation between +nodes. + +Every agent or mapped agent instance consumes model tokens. Scheduling, joins, +condition evaluation, and allowlisted transforms do not call a model. A wider +graph can therefore increase cost even when its critical path is shorter. + +## 1. Data and control edges + +![Artifact, stream, condition, and loop-back edge contracts](../assets/diagrams/graph-edge-classification.svg) + +Topology layer: an artifact edge carries one completed envelope. A stream edge +opens the matching mapped successor as soon as one item succeeds. A condition +edge selects a route. A loop-back edge stays inside a bounded loop. + +Contract layer: `edges[].type` is one of `sequence`, `artifact`, `stream`, +`condition`, or `loop-back`. Conditions are fixed vocabulary. Workflow JSON +cannot contain an expression evaluator. + +## 2. Contract-bound nodes + +An agent node has guidance and an optional payload contract. The runtime passes +only typed predecessor envelopes and the mapped item, when present. A node can +request a logical `economy`, `standard`, or `judgment` tier. It cannot name a +provider, model, tool, command, environment value, or reasoning effort. + +The operator owns concrete model selection in an execution profile: + +```json +{ + "schema_version": "1.0.0", + "profile_id": "local-codex-routing", + "tiers": { + "economy": { "model": "operator-economy-model", "reasoning_effort": "low" }, + "standard": { "model": "operator-standard-model", "reasoning_effort": "medium" }, + "judgment": { "model": "operator-judgment-model", "reasoning_effort": "high" } + } +} +``` + +Replace the three operator-owned model identifiers with models available in +your Codex installation. The profile is validated and snapshotted by digest. +`--execution-profile` cannot be combined with global `--model` or +`--reasoning-effort`, and resume uses the stored snapshot. + +## 3. Diamonds and deterministic reduction + +![Barrier diamond compared with an item streaming pipeline](../assets/diagrams/diamond-versus-pipeline.svg) + +A diamond fans one artifact into independent lenses and reduces their outputs. +Use `all` when every lens is mandatory, first-success `any` when one sufficient +answer is enough, and `quorum` when a declared threshold is the evidence rule. +Already-started branches continue to settle after `any` chooses a winner, so +the evidence trace does not hide late failures. + +Use a transform node for mechanical reduction. The allowlist is `select`, +`flatten`, `deduplicate`, `sort`, `limit`, `group`, and bounded `cartesian`. +Transforms operate on JSON values only. + +## 4. Barriers and streaming pipelines + +A normal artifact edge is a barrier at the logical-node boundary. A `stream` +edge lets one mapped successor start per successful predecessor instance. One +mapped stage can have only one stream predecessor. Map size is at most 32, +stream depth is at most four, concurrency is at most four, and a run is capped +at 128 dynamic instances or provider attempts. + +Stable fan-out identity comes from an RFC 6901 pointer into each item. The +runtime records the stable key and item digest separately. Resume recreates +missing instances without repeating completed identities. + +## 5. Conditional routing and diverse-lens quorum + +![Conditional routes entering a diverse-lens quorum](../assets/diagrams/routing-and-quorum.svg) + +Routing decides relevance. Quorum decides sufficiency. Keep those policies in +separate nodes so an omitted route cannot silently lower the verification +threshold. Grouped quorum can require representation from named lens groups in +addition to an overall threshold. + +`collect` failure handling is valid only for read or control work that feeds an +explicit threshold join. Writers and unhandled failures remain fail-closed. A +quorum fails as soon as its remaining inputs cannot satisfy the threshold. + +## 6. Retry and failure containment + +Each provider attempt has a fresh session identifier and a typed envelope. +After retry exhaustion, the failed envelope remains in the trace. A workflow +does not convert an exception into success. Writer nodes remain globally +serialized in the isolated worktree, and a writer waits for active readers to +drain. + +## 7. Bounded and until-dry cycles + +![Until-dry convergence across all previously seen keys](../assets/diagrams/until-dry-convergence.svg) + +A bounded loop stops after at most five rounds. An `until-dry` loop also stops +when a round yields no globally unseen stable key. The seen set includes every +previous decision, including rejected findings. This prevents a rejected item +from being rediscovered forever under a different round. + +## 8. Proposal, validation, activation, execution, evidence + +![Human-activated workflow lifecycle](../assets/diagrams/human-activated-workflow-lifecycle.svg) + +Generate a draft: + +```bash +./scripts/rae.sh graph workflow propose \ + --project-root /path/to/target-repository \ + --task "Design a bounded review topology for this repository" \ + --base-workflow graph-native-default \ + --actor "operator-name" \ + --rationale "Draft for topology review" +``` + +The proposal session is read-only and ephemeral. RAE permits one correction +attempt after local validation, then stores only a valid optimistic-lock draft. +The command does not activate or execute it. + +Review and activate an exact revision: + +```bash +./scripts/rae.sh graph workflow validate \ + --project-root /path/to/target-repository \ + --workflow graph-native-default --revision 2 + +./scripts/rae.sh graph workflow diff \ + --project-root /path/to/target-repository \ + --workflow graph-native-default --from 1 --to 2 + +./scripts/rae.sh graph workflow activate \ + --project-root /path/to/target-repository \ + --workflow graph-native-default --revision 2 \ + --digest \ + --actor "operator-name" \ + --rationale "Reviewed contracts, bounds, and writer path" +``` + +Activation affects future runs only. Existing runs keep their immutable 2.0 or +2.1 snapshot. + +## Six topology recipes + +The maintained recipe files are under +`packages/orchestration/workflows/recipes/`. Validate a recipe before using it, +then pass the same file to `agent run`. + +### Route auditing + +`route-audit.workflow.json` maps a bounded route inventory, applies specialist +checks, and joins all results before verification. + +```bash +./scripts/rae.sh graph workflow validate --project-root "$PWD" \ + --workflow-file packages/orchestration/workflows/recipes/route-audit.workflow.json +./scripts/rae.sh agent run --project-root /path/to/target \ + --workflow "$PWD/packages/orchestration/workflows/recipes/route-audit.workflow.json" \ + --task "Audit declared routes against handlers, authorization, and tests" +``` + +### Cited research + +`cited-research.workflow.json` separates claim collection, source checking, and +quorum synthesis. Citations remain payload data, not executable references. + +```bash +./scripts/rae.sh agent run --project-root /path/to/target \ + --workflow "$PWD/packages/orchestration/workflows/recipes/cited-research.workflow.json" \ + --execution-profile /absolute/path/to/operator-profile.json \ + --task "Research the requested change and return source-backed constraints" +``` + +### Module migration + +`module-migration.workflow.json` maps read-only module analysis, forms an +ownership plan, crosses a mutation checkpoint, and uses one serialized writer. + +```bash +./scripts/rae.sh agent run --project-root /path/to/target \ + --workflow "$PWD/packages/orchestration/workflows/recipes/module-migration.workflow.json" \ + --checkpoint-policy before-mutation \ + --task "Migrate the selected modules while preserving their public contracts" +``` + +### Adversarial review + +`adversarial-review.workflow.json` uses independent safety, contract, test, and +scope lenses with a three-of-four quorum. + +```bash +./scripts/rae.sh agent run --project-root /path/to/target \ + --workflow "$PWD/packages/orchestration/workflows/recipes/adversarial-review.workflow.json" \ + --task "Review the proposed design for blocking risks" +``` + +### Scheduled ecosystem scanning + +`ecosystem-scan.workflow.json` maps a bounded package inventory and performs a +deterministic deduplication before verification. Scheduling belongs to the +operator's local scheduler; RAE itself does not install a timer. + +```bash +./scripts/rae.sh agent run --project-root /path/to/target \ + --workflow "$PWD/packages/orchestration/workflows/recipes/ecosystem-scan.workflow.json" \ + --task "Scan the declared ecosystem snapshot for actionable compatibility changes" +``` + +### Unknown-size discovery + +`unknown-size-discovery.workflow.json` uses an until-dry loop with a 32-item +round bound and five-round ceiling. + +```bash +./scripts/rae.sh agent run --project-root /path/to/target \ + --workflow "$PWD/packages/orchestration/workflows/recipes/unknown-size-discovery.workflow.json" \ + --task "Discover and inspect all previously unseen integration points" +``` + +## Evidence limits + +The deterministic topology benchmark measures event order, fixture critical +path, and barrier idle time. It does not establish model quality or a universal +speed advantage. Provider results, hardware, repository shape, and task +difficulty remain outside that fixture's claim. + +## Source note + +- [Codex subagents](https://learn.chatgpt.com/docs/agent-configuration/subagents) +- [Codex non-interactive mode](https://learn.chatgpt.com/docs/non-interactive-mode) +- [Codex Git worktrees](https://learn.chatgpt.com/docs/environments/git-worktrees) +- [NIST GenAI Profile](../reference/claims/bibliography.md#src-nist-genai-profile) +- [OpenAI evals guidance](../reference/claims/bibliography.md#src-openai-evals) +- [PaperBench](../reference/claims/bibliography.md#src-openai-paperbench) +- [IEEE 1012](../reference/claims/bibliography.md#src-ieee-1012) +- [Model Cards](../reference/claims/bibliography.md#src-model-cards) +- [Datasheets](../reference/claims/bibliography.md#src-datasheets) +- [Amdahl 1967](../reference/claims/bibliography.md#src-amdahl-1967) diff --git a/evals/README.md b/evals/README.md index 4b76ac2..46a108c 100644 --- a/evals/README.md +++ b/evals/README.md @@ -30,6 +30,7 @@ This directory is the umbrella’s measurement layer. - `./scripts/rae.sh eval outcome --task-bundle ... --fixture-root ... --policy ... --split ... --output-dir ... --acknowledge-provider-usage` - `./scripts/rae.sh eval compare-outcomes --baseline ... --challenger ... --output ...` - `./scripts/rae.sh eval optimize --campaign ... --baseline-evaluation ... --candidate-policy ... --candidate-evaluation ... --sealed-evaluation ... --output-dir ...` +- `./scripts/rae.sh eval improve --campaign evals/campaigns/autonomous-policy-improvement.v2.json ...` - `./scripts/rae.sh release-gate --benchmark-card ... --run-card ... --regression-report ... --ledger ... --output ...` - `./evals/harness/run-local.sh validate` - `./evals/harness/run-local.sh suite ` @@ -71,3 +72,10 @@ aggregates and pairs raw challenger reports against the actual incumbent, requires an exact evaluator manifest, retains every decision, requires identical development task matrices plus a distinct held-out task-matrix digest, and cannot promote a policy automatically. + +`eval improve` is the v2 evaluator-owned wrapper around that bounded campaign. +It permits at most ten candidates, freezes task matrices, evaluator and judge +code, runtime envelope, payload contracts, and the trusted manifest. Candidate +policies remain data-only and may vary only the declared roles, guidance, safe +topology nodes and edges, joins, or loop bounds. A recommendation is an +append-only lineage artifact, never an activation. diff --git a/evals/campaigns/autonomous-policy-improvement.v2.json b/evals/campaigns/autonomous-policy-improvement.v2.json new file mode 100644 index 0000000..dad987e --- /dev/null +++ b/evals/campaigns/autonomous-policy-improvement.v2.json @@ -0,0 +1,63 @@ +{ + "campaign_id": "autonomous-policy-improvement-v2", + "campaign_version": 2, + "status": "experimental", + "max_iterations": 10, + "baseline_policy_path": "packages/orchestration/policies/default.autonomous-policy.json", + "frozen_surfaces": [ + "task_matrices", + "evaluator_and_judges", + "runtime_envelope", + "payload_contracts", + "trusted_manifest" + ], + "candidate_change_allowlist": [ + "roles", + "guidance", + "safe_nodes", + "edges", + "joins", + "loop_bounds" + ], + "trusted_paths": [ + "evals/schemas/outcome-task-spec.schema.json", + "evals/schemas/outcome-task-bundle.schema.json", + "evals/schemas/outcome-report.schema.json", + "evals/schemas/outcome-comparison.schema.json", + "evals/scripts/lib/outcome_eval.py", + "evals/scripts/lib/outcome_comparison.py", + "evals/scripts/lib/outcome_rae.py", + "evals/scripts/lib/outcome_resources.py", + "evals/scripts/lib/policy_optimizer.py", + "evals/scripts/lib/policy_optimizer_evidence.py", + "evals/scripts/lib/policy_optimizer_policy.py", + "evals/scripts/lib/policy_improvement_campaign.py", + "evals/scripts/run_outcome_benchmark.py", + "evals/scripts/compare_outcome_reports.py", + "evals/scripts/optimize_harness.py", + "evals/scripts/improve_harness.py", + "evals/schemas/improvement-campaign.schema.json", + "packages/orchestration/contracts/autonomous-policy.schema.json", + "packages/orchestration/contracts/workflows/workflow-v2.schema.json", + "packages/orchestration/contracts/workflows/node-envelope-v2.schema.json", + "packages/orchestration/workflows/graph-native-default.workflow.json", + "packages/orchestration/scripts/pipeline/lib/workflow-contract.mjs", + "packages/orchestration/scripts/pipeline/lib/workflow-scheduler.mjs", + "scripts/lib/runtime.sh", + "evals/datasets/autonomous-outcomes/core.task-bundle.json", + "evals/fixtures/autonomous-outcomes/compile-repair/README.md", + "evals/fixtures/autonomous-outcomes/compile-repair/app.py", + "evals/fixtures/autonomous-outcomes/compile-repair/tests/test_app.py", + "evals/fixtures/autonomous-outcomes/logic-regression/README.md", + "evals/fixtures/autonomous-outcomes/logic-regression/calculator.py", + "evals/fixtures/autonomous-outcomes/logic-regression/tests/test_calculator.py", + "evals/fixtures/autonomous-outcomes/scope-stress/README.md", + "evals/fixtures/autonomous-outcomes/scope-stress/normalizer.py", + "evals/fixtures/autonomous-outcomes/scope-stress/tests/test_normalizer.py" + ], + "resource_budget": { + "max_agent_duration_seconds": 7200, + "max_total_tokens": 1000000, + "max_agent_calls": 80 + } +} diff --git a/evals/campaigns/autonomous-policy.experimental.json b/evals/campaigns/autonomous-policy.experimental.json index 275aead..6cabfb1 100644 --- a/evals/campaigns/autonomous-policy.experimental.json +++ b/evals/campaigns/autonomous-policy.experimental.json @@ -15,9 +15,14 @@ "evals/scripts/lib/policy_optimizer.py", "evals/scripts/lib/policy_optimizer_evidence.py", "evals/scripts/lib/policy_optimizer_policy.py", + "evals/scripts/lib/policy_improvement_campaign.py", "evals/scripts/run_outcome_benchmark.py", "evals/scripts/compare_outcome_reports.py", "evals/scripts/optimize_harness.py", + "evals/scripts/improve_harness.py", + "evals/schemas/improvement-campaign.schema.json", + "packages/orchestration/contracts/autonomous-policy.schema.json", + "scripts/lib/runtime.sh", "evals/datasets/autonomous-outcomes/core.task-bundle.json", "evals/fixtures/autonomous-outcomes/compile-repair/README.md", "evals/fixtures/autonomous-outcomes/compile-repair/app.py", diff --git a/evals/datasets/graph-context/README.md b/evals/datasets/graph-context/README.md new file mode 100644 index 0000000..ec8faac --- /dev/null +++ b/evals/datasets/graph-context/README.md @@ -0,0 +1,24 @@ +# Frozen graph-context retrieval tasks + +`graph-context-held-out.json` contains 50 repository-localization tasks frozen +on 2026-07-29. Each task has a natural-language query and one or more expected +repository paths. + +Run the dependency-free comparison from the repository root: + +```bash +npm --prefix packages/orchestration run benchmark:graph-context -- \ + --project-root "$PWD" \ + --output /tmp/rae-graph-context-result.json +``` + +The runner compares path-only current context, lexical retrieval, lexical plus +the repository/evidence graph, and graph retrieval plus promoted memory. It +records Recall@10, estimated context tokens, retrieval latency, projection +time, stale-context rate, leakage, agent calls, and cost. + +This retrieval-only task set does not execute a provider and therefore cannot +measure held-out task pass count. The result keeps that field `null` and cannot +move graph execution out of experimental status. Provider-backed task success, +100,000-node latency, and 10,000-file projection fixtures remain separate +release evidence requirements. diff --git a/evals/datasets/graph-context/graph-context-held-out.json b/evals/datasets/graph-context/graph-context-held-out.json new file mode 100644 index 0000000..91c5412 --- /dev/null +++ b/evals/datasets/graph-context/graph-context-held-out.json @@ -0,0 +1,59 @@ +{ + "schema_version": "1.0.0", + "dataset_id": "graph-context-held-out-v1", + "split": "held-out", + "frozen_at": "2026-07-29", + "task_count": 50, + "tasks": [ + { "id": "GC-001", "query": "umbrella CLI command dispatch and runtime checks", "expected_paths": ["scripts/rae.sh"] }, + { "id": "GC-002", "query": "autonomous command option parsing and help", "expected_paths": ["packages/orchestration/scripts/pipeline/autonomous.mjs"] }, + { "id": "GC-003", "query": "phase execution provider artifact persistence", "expected_paths": ["packages/orchestration/scripts/pipeline/lib/phase-executor.mjs"] }, + { "id": "GC-004", "query": "autonomous run lifecycle request resume lock", "expected_paths": ["packages/orchestration/scripts/pipeline/lib/autonomous-lifecycle.mjs"] }, + { "id": "GC-005", "query": "autonomous workflow actions checkpoints completion", "expected_paths": ["packages/orchestration/scripts/pipeline/lib/autonomous-actions.mjs"] }, + { "id": "GC-006", "query": "phase prompt schemas ownership gate contract", "expected_paths": ["packages/orchestration/scripts/pipeline/lib/autonomous-phase-contract.mjs"] }, + { "id": "GC-007", "query": "append bounded trace events operator projection", "expected_paths": ["packages/orchestration/scripts/pipeline/lib/trace.mjs"] }, + { "id": "GC-008", "query": "must requirement traceability coverage ledger", "expected_paths": ["packages/orchestration/scripts/pipeline/lib/traceability.mjs"] }, + { "id": "GC-009", "query": "quality gate recording phase decision", "expected_paths": ["packages/orchestration/scripts/pipeline/lib/gates.mjs"] }, + { "id": "GC-010", "query": "atomic pipeline JSON state path containment", "expected_paths": ["packages/orchestration/scripts/pipeline/lib/state.mjs"] }, + { "id": "GC-011", "query": "operator durable run discovery public projection", "expected_paths": ["packages/orchestration/operator/lib/runs.mjs"] }, + { "id": "GC-012", "query": "operator loopback HTTP API routing", "expected_paths": ["packages/orchestration/operator/server.mjs"] }, + { "id": "GC-013", "query": "operator bearer origin host security", "expected_paths": ["packages/orchestration/operator/lib/security.mjs"] }, + { "id": "GC-014", "query": "operator child process start resume interrupt", "expected_paths": ["packages/orchestration/operator/lib/control.mjs"] }, + { "id": "GC-015", "query": "brief requirements constraints JSON schema", "expected_paths": ["packages/orchestration/contracts/artifacts/brief.schema.json"] }, + { "id": "GC-016", "query": "execution plan tasks file ownership verification commands schema", "expected_paths": ["packages/orchestration/contracts/artifacts/execution-plan.schema.json"] }, + { "id": "GC-017", "query": "build report outputs coverage schema", "expected_paths": ["packages/orchestration/contracts/artifacts/build-report.schema.json"] }, + { "id": "GC-018", "query": "quality report evidence bundle violations schema", "expected_paths": ["packages/orchestration/contracts/artifacts/quality-report.schema.json"] }, + { "id": "GC-019", "query": "release readiness conditions approvals schema", "expected_paths": ["packages/orchestration/contracts/artifacts/release-readiness.schema.json"] }, + { "id": "GC-020", "query": "execution trace event JSONL contract", "expected_paths": ["packages/orchestration/contracts/artifacts/execution-trace.schema.json"] }, + { "id": "GC-021", "query": "autonomous policy data phase guidance schema", "expected_paths": ["packages/orchestration/contracts/autonomous-policy.schema.json"] }, + { "id": "GC-022", "query": "quality gate contract criteria blocking failures", "expected_paths": ["packages/orchestration/contracts/quality-gate.schema.json"] }, + { "id": "GC-023", "query": "operator console API documentation", "expected_paths": ["packages/orchestration/operator/README.md"] }, + { "id": "GC-024", "query": "orchestration execution model ten stages", "expected_paths": ["packages/orchestration/README.md"] }, + { "id": "GC-025", "query": "repository architecture system overview", "expected_paths": ["docs/reference/architecture/system-overview.md"] }, + { "id": "GC-026", "query": "repository module ownership boundaries", "expected_paths": ["docs/reference/architecture/module-boundaries.md"] }, + { "id": "GC-027", "query": "umbrella CLI reference commands", "expected_paths": ["docs/reference/cli/umbrella.md"] }, + { "id": "GC-028", "query": "orchestration CLI reference stages", "expected_paths": ["docs/reference/cli/orchestration.md"] }, + { "id": "GC-029", "query": "artifact schemas contract documentation", "expected_paths": ["docs/reference/contracts/artifact-schemas.md"] }, + { "id": "GC-030", "query": "human checkpoint decision contract", "expected_paths": ["docs/reference/contracts/human-checkpoints.md"] }, + { "id": "GC-031", "query": "quality gate reference contract", "expected_paths": ["docs/reference/contracts/quality-gates.md"] }, + { "id": "GC-032", "query": "provenance evidence requirements invariant", "expected_paths": ["docs/reference/invariants/provenance-requirements.md"] }, + { "id": "GC-033", "query": "deterministic reproducibility contracts invariant", "expected_paths": ["docs/reference/invariants/determinism-contracts.md"] }, + { "id": "GC-034", "query": "security safety boundaries invariant", "expected_paths": ["docs/reference/invariants/safety-boundaries.md"] }, + { "id": "GC-035", "query": "claims ledger publication evidence", "expected_paths": ["docs/reference/claims/claims-ledger.md"] }, + { "id": "GC-036", "query": "claims evidence index mappings", "expected_paths": ["docs/reference/claims/evidence-index.md"] }, + { "id": "GC-037", "query": "benchmark contamination assumptions register", "expected_paths": ["docs/reference/claims/assumptions-register.md"] }, + { "id": "GC-038", "query": "research source bibliography", "expected_paths": ["docs/reference/claims/bibliography.md"] }, + { "id": "GC-039", "query": "repo audit benchmark card", "expected_paths": ["evals/benchmarks/repo-audit-core.benchmark-card.json"] }, + { "id": "GC-040", "query": "tool selection benchmark card", "expected_paths": ["evals/benchmarks/tool-selection-core.benchmark-card.json"] }, + { "id": "GC-041", "query": "long horizon task specifications", "expected_paths": ["evals/datasets/long-horizon/long-horizon-core.task-specs.json"] }, + { "id": "GC-042", "query": "scoped fix task specifications", "expected_paths": ["evals/datasets/scoped-fix/scoped-fix-core.task-specs.json"] }, + { "id": "GC-043", "query": "autonomous outcomes task bundle", "expected_paths": ["evals/datasets/autonomous-outcomes/core.task-bundle.json"] }, + { "id": "GC-044", "query": "compile repair evaluation fixture", "expected_paths": ["evals/fixtures/autonomous-outcomes/compile-repair/app.py"] }, + { "id": "GC-045", "query": "logic regression calculator fixture", "expected_paths": ["evals/fixtures/autonomous-outcomes/logic-regression/calculator.py"] }, + { "id": "GC-046", "query": "scope stress normalizer fixture", "expected_paths": ["evals/fixtures/autonomous-outcomes/scope-stress/normalizer.py"] }, + { "id": "GC-047", "query": "evaluation local harness commands", "expected_paths": ["evals/harness/run-local.sh"] }, + { "id": "GC-048", "query": "frozen evaluation suite harness", "expected_paths": ["evals/harness/run-frozen-suite.sh"] }, + { "id": "GC-049", "query": "programmatic router judge", "expected_paths": ["evals/judges/programmatic-router-judge.json"] }, + { "id": "GC-050", "query": "repository map package ownership", "expected_paths": ["docs/reference/repo-map.md"] } + ] +} diff --git a/evals/harness/run-local.sh b/evals/harness/run-local.sh index 768c95b..a90cf59 100755 --- a/evals/harness/run-local.sh +++ b/evals/harness/run-local.sh @@ -23,6 +23,7 @@ Commands: compare-outcomes Compare paired outcome reports for optimizer evidence optimize Evaluate a bounded experimental policy campaign from precomputed evidence + improve Evaluate a sealed evaluator-owned RAE v2 improvement campaign suite Execute all frozen benchmark families for dev and held-out splits under evals/results calibrate Run judge calibration release-gate Evaluate release-blocking gates for a run card @@ -73,6 +74,7 @@ run_doctor() { check_file "outcome-runner" "$ROOT_DIR/evals/scripts/run_outcome_benchmark.py" || failed=1 check_file "outcome-compare" "$ROOT_DIR/evals/scripts/compare_outcome_reports.py" || failed=1 check_file "policy-optimizer" "$ROOT_DIR/evals/scripts/optimize_harness.py" || failed=1 + check_file "policy-improvement" "$ROOT_DIR/evals/scripts/improve_harness.py" || failed=1 check_file "release-gate" "$ROOT_DIR/evals/scripts/release_gate.py" || failed=1 if [[ "$failed" -ne 0 ]]; then @@ -109,6 +111,9 @@ main() { optimize) "$PYTHON_BIN" "$ROOT_DIR/evals/scripts/optimize_harness.py" "$@" ;; + improve) + "$PYTHON_BIN" "$ROOT_DIR/evals/scripts/improve_harness.py" "$@" + ;; suite) "$BASH_BIN" "$ROOT_DIR/evals/harness/run-frozen-suite.sh" "$@" ;; diff --git a/evals/schemas/improvement-campaign.schema.json b/evals/schemas/improvement-campaign.schema.json new file mode 100644 index 0000000..f87da8a --- /dev/null +++ b/evals/schemas/improvement-campaign.schema.json @@ -0,0 +1,18 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "EvaluatorOwnedImprovementCampaignV2", + "type": "object", + "required": ["campaign_id", "campaign_version", "status", "max_iterations", "baseline_policy_path", "frozen_surfaces", "candidate_change_allowlist", "trusted_paths"], + "properties": { + "campaign_id": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]*$"}, + "campaign_version": {"const": 2}, + "status": {"const": "experimental"}, + "max_iterations": {"type": "integer", "minimum": 1, "maximum": 10}, + "baseline_policy_path": {"type": "string", "minLength": 1}, + "frozen_surfaces": {"type": "array", "uniqueItems": true, "minItems": 5, "maxItems": 5, "items": {"enum": ["task_matrices", "evaluator_and_judges", "runtime_envelope", "payload_contracts", "trusted_manifest"]}}, + "candidate_change_allowlist": {"type": "array", "uniqueItems": true, "minItems": 6, "maxItems": 6, "items": {"enum": ["roles", "guidance", "safe_nodes", "edges", "joins", "loop_bounds"]}}, + "trusted_paths": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"type": "string", "minLength": 1}}, + "resource_budget": {"type": "object"} + }, + "additionalProperties": false +} diff --git a/evals/scripts/improve_harness.py b/evals/scripts/improve_harness.py new file mode 100644 index 0000000..b68b9ff --- /dev/null +++ b/evals/scripts/improve_harness.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +"""Run a bounded evaluator-owned RAE v2 improvement campaign from sealed evidence.""" + +from __future__ import annotations + +import argparse + +from common import load_json +from lib.policy_improvement_campaign import validate_improvement_campaign +from optimize_harness import _path +from optimize_harness import main as optimize_main + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--campaign", required=True) + args, _unknown = parser.parse_known_args() + try: + validate_improvement_campaign(load_json(_path(args.campaign, "campaign"))) + except ValueError as exc: + raise SystemExit(f"invalid improvement campaign: {exc}") from exc + return optimize_main() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/evals/scripts/lib/outcome_eval.py b/evals/scripts/lib/outcome_eval.py index 8310f65..46a1504 100644 --- a/evals/scripts/lib/outcome_eval.py +++ b/evals/scripts/lib/outcome_eval.py @@ -85,9 +85,14 @@ def _python_unittest(workspace: pathlib.Path, _task: dict[str, Any]) -> list[str "evals/scripts/lib/policy_optimizer.py", "evals/scripts/lib/policy_optimizer_evidence.py", "evals/scripts/lib/policy_optimizer_policy.py", + "evals/scripts/lib/policy_improvement_campaign.py", "evals/scripts/run_outcome_benchmark.py", "evals/scripts/compare_outcome_reports.py", "evals/scripts/optimize_harness.py", + "evals/scripts/improve_harness.py", + "evals/schemas/improvement-campaign.schema.json", + "packages/orchestration/contracts/autonomous-policy.schema.json", + "scripts/lib/runtime.sh", ) diff --git a/evals/scripts/lib/policy_improvement_campaign.py b/evals/scripts/lib/policy_improvement_campaign.py new file mode 100644 index 0000000..44723cf --- /dev/null +++ b/evals/scripts/lib/policy_improvement_campaign.py @@ -0,0 +1,31 @@ +"""Validation for the evaluator-owned RAE v2 improvement campaign.""" + +from __future__ import annotations + +from typing import Any + +from lib.policy_optimizer_policy import IMPROVEMENT_CHANGE_ALLOWLIST, validate_campaign + +_FROZEN_SURFACES = frozenset( + { + "task_matrices", + "evaluator_and_judges", + "runtime_envelope", + "payload_contracts", + "trusted_manifest", + } +) + + +def validate_improvement_campaign(campaign: object) -> dict[str, Any]: + """Require the complete v2 freeze contract before offline optimization.""" + validated = validate_campaign(campaign) + if validated.get("campaign_version") != 2: + raise ValueError("improvement campaign_version must be 2") + if frozenset(validated.get("frozen_surfaces", ())) != _FROZEN_SURFACES: + raise ValueError("improvement campaign must freeze every evaluator-owned surface") + if frozenset(validated.get("candidate_change_allowlist", ())) != IMPROVEMENT_CHANGE_ALLOWLIST: + raise ValueError( + "improvement campaign candidate changes exceed the evaluator-owned allowlist" + ) + return validated diff --git a/evals/scripts/lib/policy_optimizer.py b/evals/scripts/lib/policy_optimizer.py index 8265912..489713e 100644 --- a/evals/scripts/lib/policy_optimizer.py +++ b/evals/scripts/lib/policy_optimizer.py @@ -6,8 +6,10 @@ from __future__ import annotations +import json import pathlib -from typing import Any +from dataclasses import dataclass +from typing import Any, cast from common import append_jsonl, dump_json, iso_timestamp @@ -25,10 +27,59 @@ policy_digest, trusted_manifest, validate_campaign, + validate_candidate_policy_change, validate_policy, ) +@dataclass(frozen=True) +class _CampaignRun: + """Keep campaign controls together after the public keyword boundary.""" + + proposer: Proposer + evaluator: Evaluator + trusted_paths: list[pathlib.Path] + output_dir: pathlib.Path + max_iterations: int + resource_budget: dict[str, float] | None + sealed_evaluator: Evaluator | None + candidate_change_allowlist: object + + +def _campaign_run( + proposer: Proposer, + evaluator: Evaluator, + trusted_paths: list[pathlib.Path], + output_dir: pathlib.Path, + controls: dict[str, object], +) -> _CampaignRun: + expected_controls = { + "max_iterations", + "resource_budget", + "sealed_evaluator", + "candidate_change_allowlist", + } + unexpected_controls = sorted(set(controls) - expected_controls) + if unexpected_controls: + raise TypeError( + f"optimize_campaign() got an unexpected keyword argument {unexpected_controls[0]!r}" + ) + if "max_iterations" not in controls: + raise TypeError( + "optimize_campaign() missing 1 required keyword-only argument: 'max_iterations'" + ) + return _CampaignRun( + proposer=proposer, + evaluator=evaluator, + trusted_paths=trusted_paths, + output_dir=output_dir, + max_iterations=cast(int, controls["max_iterations"]), + resource_budget=cast(dict[str, float] | None, controls.get("resource_budget")), + sealed_evaluator=cast(Evaluator | None, controls.get("sealed_evaluator")), + candidate_change_allowlist=controls.get("candidate_change_allowlist"), + ) + + def _record_event( lineage: list[dict[str, Any]], lineage_path: pathlib.Path, event: dict[str, Any] ) -> None: @@ -36,6 +87,22 @@ def _record_event( append_jsonl(lineage_path, event) +def recover_lineage(lineage_path: pathlib.Path) -> list[dict[str, Any]]: + """Read append-only lineage without treating a partial record as evidence.""" + if not lineage_path.exists(): + return [] + recovered: list[dict[str, Any]] = [] + for line_number, line in enumerate(lineage_path.read_text(encoding="utf-8").splitlines(), 1): + try: + event = json.loads(line) + except json.JSONDecodeError as exc: + raise ValueError(f"lineage recovery failed at line {line_number}") from exc + if not isinstance(event, dict): + raise ValueError(f"lineage recovery found a non-object event at line {line_number}") + recovered.append(event) + return recovered + + def _campaign_baseline( baseline_policy: Policy, evaluator: Evaluator, @@ -59,11 +126,13 @@ def _candidate_or_rejection( lineage: list[dict[str, Any]], iteration: int, lineage_path: pathlib.Path, + candidate_change_allowlist: object, ) -> tuple[Policy | None, str]: candidate = proposer(incumbent, lineage) candidate_id = f"candidate-{iteration:02d}" try: validate_policy(candidate) + validate_candidate_policy_change(incumbent, candidate, candidate_change_allowlist) except ValueError as exc: _record_event( lineage, @@ -261,25 +330,25 @@ def optimize_campaign( evaluator: Evaluator, trusted_paths: list[pathlib.Path], output_dir: pathlib.Path, - max_iterations: int, - resource_budget: dict[str, float] | None = None, - sealed_evaluator: Evaluator | None = None, + **controls: object, ) -> dict[str, Any]: """Run a bounded single-challenger campaign with fully retained lineage.""" + campaign = _campaign_run(proposer, evaluator, trusted_paths, output_dir, controls) state, lineage_path = _start_campaign( - baseline_policy, evaluator, trusted_paths, output_dir, max_iterations + baseline_policy, + campaign.evaluator, + campaign.trusted_paths, + campaign.output_dir, + campaign.max_iterations, ) - _run_iterations( + _run_iterations(state, lineage_path, campaign) + return _finish_campaign( + baseline_policy, state, - lineage_path, - proposer, - evaluator, - resource_budget, - output_dir, - max_iterations, - trusted_paths, + campaign.trusted_paths, + campaign.output_dir, + campaign.sealed_evaluator, ) - return _finish_campaign(baseline_policy, state, trusted_paths, output_dir, sealed_evaluator) def _start_campaign( @@ -309,49 +378,51 @@ def _start_campaign( return state, output_dir / "lineage.jsonl" +def _record_blocked_iteration( + state: dict[str, Any], + lineage_path: pathlib.Path, + iteration: int, + reason: str, + candidate_id: str | None = None, +) -> None: + event: dict[str, Any] = {"iteration": iteration, "decision": "blocked", "reason": reason} + if candidate_id is not None: + event["candidate_id"] = candidate_id + _record_event(state["lineage"], lineage_path, event) + + def _run_iterations( state: dict[str, Any], lineage_path: pathlib.Path, - proposer: Proposer, - evaluator: Evaluator, - resource_budget: dict[str, float] | None, - output_dir: pathlib.Path, - max_iterations: int, - trusted_paths: list[pathlib.Path], + campaign: _CampaignRun, ) -> None: - for iteration in range(1, max_iterations + 1): - if trusted_manifest(trusted_paths) != state["initial_manifest"]: - _record_event( - state["lineage"], - lineage_path, - { - "iteration": iteration, - "decision": "blocked", - "reason": "evaluator-integrity-drift", - }, - ) + for iteration in range(1, campaign.max_iterations + 1): + if trusted_manifest(campaign.trusted_paths) != state["initial_manifest"]: + _record_blocked_iteration(state, lineage_path, iteration, "evaluator-integrity-drift") break candidate, candidate_id = _candidate_or_rejection( - proposer, state["incumbent"], state["lineage"], iteration, lineage_path + campaign.proposer, + state["incumbent"], + state["lineage"], + iteration, + lineage_path, + campaign.candidate_change_allowlist, ) if candidate is None: continue - dump_json(output_dir / "candidates" / f"{candidate_id}.policy.json", candidate) + dump_json(campaign.output_dir / "candidates" / f"{candidate_id}.policy.json", candidate) budget_blocked, evaluation, decision = _evaluate_iteration( - state, candidate, evaluator, resource_budget + state, candidate, campaign.evaluator, campaign.resource_budget ) if budget_blocked: - _record_event( - state["lineage"], + _record_blocked_iteration( + state, lineage_path, - { - "iteration": iteration, - "candidate_id": candidate_id, - "decision": "blocked", - "reason": "budget-exceeded-or-incomplete-measurement", - }, + iteration, + "budget-exceeded-or-incomplete-measurement", + candidate_id, ) - dump_json(output_dir / "evaluations" / f"{candidate_id}.json", evaluation) + dump_json(campaign.output_dir / "evaluations" / f"{candidate_id}.json", evaluation) break if decision is None: raise RuntimeError("policy search did not produce a decision") @@ -363,13 +434,14 @@ def _run_iterations( candidate, evaluation, decision, - output_dir, + campaign.output_dir, ) __all__ = [ "optimize_campaign", "policy_digest", + "recover_lineage", "trusted_manifest", "validate_campaign", "validate_policy", diff --git a/evals/scripts/lib/policy_optimizer_policy.py b/evals/scripts/lib/policy_optimizer_policy.py index 67f1289..83ca448 100644 --- a/evals/scripts/lib/policy_optimizer_policy.py +++ b/evals/scripts/lib/policy_optimizer_policy.py @@ -6,7 +6,7 @@ import json import pathlib import re -from collections.abc import Callable +from collections.abc import Callable, Iterable from typing import Any from common import ROOT, is_within_directory @@ -90,6 +90,9 @@ MAX_TASK_ATTEMPTS = 12 MIN_PAIRED_WINS = 2 MIN_IMPROVEMENT = 0.05 +IMPROVEMENT_CHANGE_ALLOWLIST = frozenset( + {"roles", "guidance", "safe_nodes", "edges", "joins", "loop_bounds"} +) def sha256_file(path: pathlib.Path) -> str: @@ -163,6 +166,33 @@ def validate_policy(policy: Policy) -> None: _validate_policy_inputs(inputs) +def validate_candidate_policy_change( + incumbent: Policy, candidate: Policy, allowed_changes: object = None +) -> None: + """Keep experimental candidates inside the data-only policy surface. + + The current autonomous-policy schema exposes guidance and safe artifact + topology only. The v2 campaign labels that bounded surface with the + broader vocabulary used by future evaluator-owned policy adapters; it + cannot alter runtime commands, judges, evaluator code, or activation. + """ + if allowed_changes is None: + allowlist = IMPROVEMENT_CHANGE_ALLOWLIST + elif isinstance(allowed_changes, Iterable): + allowlist = frozenset(allowed_changes) + else: + raise ValueError("candidate change allowlist must be an iterable") + if allowlist != IMPROVEMENT_CHANGE_ALLOWLIST: + raise ValueError( + "candidate change allowlist must exactly match the evaluator-owned allowlist" + ) + validate_policy(incumbent) + validate_policy(candidate) + changed = {key for key in _POLICY_KEYS if incumbent.get(key) != candidate.get(key)} + if changed - {"policy_id", "phase_guidance", "phase_inputs"}: + raise ValueError("candidate changes an evaluator-forbidden policy field") + + def _validate_campaign_header(campaign: object) -> dict[str, Any]: if not isinstance(campaign, dict): raise ValueError("campaign must be a JSON object") @@ -173,7 +203,12 @@ def _validate_campaign_header(campaign: object) -> dict[str, Any]: "baseline_policy_path", "trusted_paths", } - allowed = required | {"resource_budget"} + allowed = required | { + "resource_budget", + "campaign_version", + "frozen_surfaces", + "candidate_change_allowlist", + } if not _has_campaign_fields(campaign, required, allowed): raise ValueError("campaign fields do not match the optimizer campaign contract") campaign_id = campaign["campaign_id"] diff --git a/evals/scripts/optimize_harness.py b/evals/scripts/optimize_harness.py index 6a78ca9..900ea02 100644 --- a/evals/scripts/optimize_harness.py +++ b/evals/scripts/optimize_harness.py @@ -136,6 +136,7 @@ def evaluate(policy: dict) -> dict: output_dir=output_dir, max_iterations=int(campaign["max_iterations"]), resource_budget=campaign.get("resource_budget"), + candidate_change_allowlist=campaign.get("candidate_change_allowlist"), ) diff --git a/evals/tests/outcome_optimizer_helpers.py b/evals/tests/outcome_optimizer_helpers.py index d9faaf6..ca955d2 100644 --- a/evals/tests/outcome_optimizer_helpers.py +++ b/evals/tests/outcome_optimizer_helpers.py @@ -35,6 +35,7 @@ task_matrix_digest = outcome_eval.task_matrix_digest trusted_judge_argv = outcome_eval.trusted_judge_argv optimize_campaign = policy_optimizer.optimize_campaign +recover_lineage = policy_optimizer.recover_lineage policy_digest = policy_optimizer.policy_digest trusted_manifest = policy_optimizer.trusted_manifest resource_usage_issues = release_gate_core._resource_usage_issues diff --git a/evals/tests/test_benchmark_contracts_core_a.py b/evals/tests/test_benchmark_contracts_core_a.py index 4baaa6d..dbc1505 100644 --- a/evals/tests/test_benchmark_contracts_core_a.py +++ b/evals/tests/test_benchmark_contracts_core_a.py @@ -3,6 +3,7 @@ import json import os import pathlib +import shutil import subprocess import sys import tempfile @@ -19,6 +20,16 @@ ) +def _trusted_bash() -> str: + bash = shutil.which("bash") + if bash is None: + raise RuntimeError("bash is required for the doctor contract test") + resolved = pathlib.Path(bash).resolve() + if not resolved.is_absolute() or not resolved.is_file(): + raise RuntimeError("resolved bash executable is not a regular absolute path") + return str(resolved) + + def test_run_benchmark_rejects_output_dir_outside_results_root() -> None: with tempfile.TemporaryDirectory(prefix="rae-benchmark-outside-") as tmp: output_dir = pathlib.Path(tmp) @@ -198,7 +209,7 @@ def test_rae_doctor_reports_missing_rg_dependency() -> None: install_path_mirror(bin_dir) # B603 rationale: fixed Bash executable and repository test entrypoint. completed = subprocess.run( # nosec B603 - ["bash", str(ROOT / "scripts/rae.sh"), "doctor"], + [_trusted_bash(), str(ROOT / "scripts/rae.sh"), "doctor"], cwd=ROOT, text=True, capture_output=True, @@ -215,7 +226,7 @@ def test_rae_doctor_reports_missing_rg_dependency() -> None: install_path_mirror(bin_dir, exclude={"rg"}) # B603 rationale: fixed Bash executable and repository test entrypoint. completed = subprocess.run( # nosec B603 - ["bash", str(ROOT / "scripts/rae.sh"), "doctor"], + [_trusted_bash(), str(ROOT / "scripts/rae.sh"), "doctor"], cwd=ROOT, text=True, capture_output=True, @@ -231,7 +242,7 @@ def test_rae_doctor_reports_missing_rg_dependency() -> None: def test_rae_worktree_help_lists_supervision_commands() -> None: # B603 rationale: fixed Bash executable and repository test entrypoint. completed = subprocess.run( # nosec B603 - ["bash", str(ROOT / "scripts/rae.sh"), "worktree", "help"], + [_trusted_bash(), str(ROOT / "scripts/rae.sh"), "worktree", "help"], cwd=ROOT, text=True, capture_output=True, diff --git a/evals/tests/test_benchmark_contracts_core_b.py b/evals/tests/test_benchmark_contracts_core_b.py index d16ea67..9fbf2cd 100644 --- a/evals/tests/test_benchmark_contracts_core_b.py +++ b/evals/tests/test_benchmark_contracts_core_b.py @@ -30,6 +30,36 @@ def _remove_gate_report_path(run_card_path: pathlib.Path) -> None: write_json(run_card_path, payload) +def _stale_required_split_benchmark() -> tuple[dict, pathlib.Path]: + benchmark_path = ROOT / "evals/benchmarks/tool-selection-core.benchmark-card.json" + benchmark = json.loads(benchmark_path.read_text(encoding="utf-8")) + benchmark = { + **benchmark, + "benchmark_id": "tool-selection-core-stale-required-split", + "version": "1.0.2", + } + path = RESULTS_ROOT / ".tmp-tool-selection-core-stale-required-split.benchmark-card.json" + write_json(path, benchmark) + return benchmark, path + + +def _write_stale_dev_fixture(output_dir: pathlib.Path, benchmark: dict) -> None: + _, _, _, _, run_card_path = write_release_gate_fixture( + output_dir / "dev-stale", + split="dev", + run_id="tool-selection-core-stale-required-split-dev", + benchmark=benchmark, + calibration_payload={ + "judge_id": "router", + "agreement_rate": 1.0, + "calibration_case_count": 4, + "status": "pass", + }, + release_gate_status="pass", + ) + _remove_gate_report_path(run_card_path) + + def test_validate_eval_metadata_discovers_generated_run_card_names() -> None: benchmark_path = ROOT / "evals/benchmarks/tool-selection-core.benchmark-card.json" benchmark = json.loads(benchmark_path.read_text(encoding="utf-8")) @@ -94,38 +124,14 @@ def test_release_gate_ignores_stale_passing_run_cards_for_required_split() -> No # The held-out gate checks for dev evidence (prior required split). A dev # run-card that declares release_gate_status: pass but is missing its gate # report file is stale and must be rejected. - benchmark_path = ROOT / "evals/benchmarks/tool-selection-core.benchmark-card.json" - benchmark = json.loads(benchmark_path.read_text(encoding="utf-8")) - benchmark = { - **benchmark, - "benchmark_id": "tool-selection-core-stale-required-split", - "version": "1.0.2", - } - temp_benchmark_path = ( - RESULTS_ROOT / ".tmp-tool-selection-core-stale-required-split.benchmark-card.json" - ) - write_json(temp_benchmark_path, benchmark) + benchmark, temp_benchmark_path = _stale_required_split_benchmark() with tempfile.TemporaryDirectory( dir=RESULTS_ROOT, prefix="release-gate-stale-required-split-" ) as tmp: output_dir = pathlib.Path(tmp) # Create the dev fixture then corrupt it by removing its gate report path. - dev_dir = output_dir / "dev-stale" - _, _, _, _, dev_run_card_path = write_release_gate_fixture( - dev_dir, - split="dev", - run_id="tool-selection-core-stale-required-split-dev", - benchmark=benchmark, - calibration_payload={ - "judge_id": "router", - "agreement_rate": 1.0, - "calibration_case_count": 4, - "status": "pass", - }, - release_gate_status="pass", - ) - _remove_gate_report_path(dev_run_card_path) + _write_stale_dev_fixture(output_dir, benchmark) # Create the held-out fixture; this is the run being gated. _, regression_path, ledger_path, _, run_card_path = write_release_gate_fixture( diff --git a/evals/tests/test_policy_optimizer_contracts.py b/evals/tests/test_policy_optimizer_contracts.py index 90f91fa..8b8ce7d 100644 --- a/evals/tests/test_policy_optimizer_contracts.py +++ b/evals/tests/test_policy_optimizer_contracts.py @@ -14,10 +14,16 @@ evaluation, optimize_campaign, policy, + recover_lineage, resource_usage_issues, ) +def require(condition: bool, message: str) -> None: + if not condition: + raise AssertionError(message) + + def test_optimizer_accepts_only_measurable_improvement_without_auto_promotion() -> None: baseline = policy() candidate = policy("candidate") @@ -186,3 +192,113 @@ def test_optimizer_rejects_out_of_range_reports_and_standalone_comparisons() -> assert "evidence_type is not accepted" in str(exc) else: raise AssertionError("standalone comparison evidence was accepted") + + +def test_improvement_campaign_hill_climbs_ten_candidates_with_paired_wins_and_held_out_seal() -> ( + None +): + baseline = policy() + candidates = [policy(f"candidate-{index}") for index in range(10)] + for index, candidate in enumerate(candidates): + candidate["phase_guidance"]["plan"] = f"bounded guidance {index}" + + def evaluator(evaluated: dict[str, Any]) -> dict[str, Any]: + score = 0.5 if evaluated["policy_id"] == "baseline" else 1.0 + return evaluation(evaluated, score, evidence_type=OUTCOME_REPORT_TYPE) + + with tempfile.TemporaryDirectory(dir=RESULTS_ROOT, prefix="rae-improve-ten-") as tmp: + output = pathlib.Path(tmp) + report = optimize_campaign( + baseline_policy=baseline, + proposer=lambda _incumbent, lineage: candidates[len(lineage)], + evaluator=evaluator, + trusted_paths=[TRUSTED_EVALUATOR_PATH], + output_dir=output, + max_iterations=10, + candidate_change_allowlist=[ + "roles", + "guidance", + "safe_nodes", + "edges", + "joins", + "loop_bounds", + ], + sealed_evaluator=lambda evaluated: evaluation( + evaluated, 1.0, split="held-out", evidence_type=OUTCOME_REPORT_TYPE + ), + ) + lineage = recover_lineage(output / "lineage.jsonl") + require(len(lineage) == 10, "campaign did not retain ten lineage events") + require(lineage[0]["decision"] == "accepted", "first improvement candidate was not accepted") + require( + all(event["decision"] in {"accepted", "rejected"} for event in lineage), + "lineage contains an unexpected decision", + ) + require(report["recommendation_status"] == "recommended", "campaign was not recommended") + require(report["automatic_promotion"] is False, "campaign promoted a policy automatically") + + +def test_improvement_candidate_forbidden_change_is_retained_as_rejection() -> None: + baseline, candidate = policy(), policy("candidate") + candidate["unexpected_runtime"] = "activate" + with tempfile.TemporaryDirectory(dir=RESULTS_ROOT, prefix="rae-improve-forbidden-") as tmp: + output = pathlib.Path(tmp) + optimize_campaign( + baseline_policy=baseline, + proposer=lambda _incumbent, _lineage: candidate, + evaluator=lambda evaluated: evaluation( + evaluated, 0.5, evidence_type=OUTCOME_REPORT_TYPE + ), + trusted_paths=[TRUSTED_EVALUATOR_PATH], + output_dir=output, + max_iterations=1, + ) + lineage = recover_lineage(output / "lineage.jsonl") + require(lineage[0]["decision"] == "rejected", "forbidden candidate was accepted") + require("policy" in lineage[0]["reason"], "rejection did not report the policy violation") + + +def test_improvement_lineage_recovery_rejects_partial_records() -> None: + with tempfile.TemporaryDirectory(dir=RESULTS_ROOT, prefix="rae-improve-recover-") as tmp: + lineage_path = pathlib.Path(tmp) / "lineage.jsonl" + lineage_path.write_text('{"iteration":1}\n{', encoding="utf-8") + try: + recover_lineage(lineage_path) + except ValueError as exc: + require("lineage recovery failed" in str(exc), "partial lineage error was unclear") + else: + raise AssertionError("partial lineage record was accepted") + + +def test_improvement_rejects_resource_regression_even_with_paired_wins() -> None: + baseline, candidate = policy(), policy("candidate") + candidate["phase_guidance"]["plan"] = "bounded guidance" + + def evaluator(evaluated: dict[str, Any]) -> dict[str, Any]: + score = 0.5 if evaluated["policy_id"] == "baseline" else 1.0 + report = evaluation(evaluated, score, evidence_type=OUTCOME_REPORT_TYPE) + if evaluated["policy_id"] == "candidate": + for result in report["repeats"][0]: + result["resource_usage"] = { + **result["resource_usage"], + "agent_duration_seconds": 100.0, + } + report["aggregate"] = aggregate_repeats(report["repeats"]) + return report + + with tempfile.TemporaryDirectory(dir=RESULTS_ROOT, prefix="rae-improve-resource-") as tmp: + output = pathlib.Path(tmp) + optimize_campaign( + baseline_policy=baseline, + proposer=lambda _incumbent, _lineage: candidate, + evaluator=evaluator, + trusted_paths=[TRUSTED_EVALUATOR_PATH], + output_dir=output, + max_iterations=1, + ) + lineage = recover_lineage(output / "lineage.jsonl") + require(lineage[0]["decision"] == "rejected", "resource regression was accepted") + require( + lineage[0]["reason"] == "challenger-hard-failure", + "resource regression was not classified as a hard failure", + ) diff --git a/mkdocs.yml b/mkdocs.yml index 3aa60b0..184f6ec 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -41,6 +41,7 @@ exclude_docs: | nav: - Home: INDEX.md - Tutorials: + - Graph Engineering with Codex and RAE: tutorials/graph-engineering-with-codex.md - Autonomous Code Change: tutorials/autonomous-code-change.md - First Pipeline: tutorials/first-pipeline.md - First Ralph Run: tutorials/first-ralph-run.md @@ -66,6 +67,7 @@ nav: - Result Ledger: reference/contracts/result-ledger.md - Quality Gates: reference/contracts/quality-gates.md - Report Types: reference/contracts/report-types.md + - Local Graph and Memory: reference/contracts/graph-memory.md - CLI: - Umbrella CLI: reference/cli/umbrella.md - Orchestration CLI: reference/cli/orchestration.md diff --git a/packages/loops/ralph/scripts/ralph_fs_txn.py b/packages/loops/ralph/scripts/ralph_fs_txn.py index 52cd3bb..8063fbf 100644 --- a/packages/loops/ralph/scripts/ralph_fs_txn.py +++ b/packages/loops/ralph/scripts/ralph_fs_txn.py @@ -983,6 +983,53 @@ def pointer_path_command(args: argparse.Namespace) -> int: return 0 +def _create_mirror_journal( + root: bytes, + runtime: bytes, + metadata_root: bytes, + transaction_dir: Path, + provider_directory: Path, +) -> tuple[Path, dict[str, Any]]: + baseline_path = transaction_dir / "baseline" + baseline_path.mkdir(mode=0o700) + mirror_path = provider_directory / WORKSPACE_NAME + mirror_path.mkdir(mode=0o700) + mirror = os.fsencode(mirror_path) + baseline_store = os.fsencode(baseline_path) + manifest = make_manifest(root, runtime) + copy_manifest(root, baseline_store, manifest) + copy_manifest(root, mirror, manifest) + transaction_id = str(uuid.uuid4()) + quarantine_root = os.path.join(runtime, b".fixing-quarantine", os.fsencode(transaction_id)) + journal = { + "format": FORMAT_VERSION, + "id": transaction_id, + "state": "mirrored", + "root": os.fsdecode(root), + "runtime": os.fsdecode(runtime), + "metadata_root": os.fsdecode(metadata_root), + "mirror": os.fsdecode(mirror), + "baseline_store": os.fsdecode(baseline_store), + "quarantine_root": os.fsdecode(quarantine_root), + "root_identity": identity(os.lstat(root)), + "runtime_identity": identity(os.lstat(runtime)), + "metadata_root_identity": identity(os.lstat(metadata_root)), + "mirror_identity": identity(os.lstat(mirror)), + "provider_directory_identity": identity(os.lstat(provider_directory)), + "baseline_store_identity": identity(os.lstat(baseline_store)), + "baseline": manifest, + "prepared": None, + "changed": [], + "promoted": [], + "active": None, + "active_started": False, + "evidence": [], + } + journal_path = transaction_dir / "journal.json" + json_dump_atomic(journal_path, journal) + return journal_path, journal + + def mirror_command(args: argparse.Namespace) -> int: root, runtime, metadata_root, pointer = caller_identity(args) if os.path.lexists(pointer): @@ -996,46 +1043,12 @@ def mirror_command(args: argparse.Namespace) -> int: os.path.realpath(tempfile.mkdtemp(prefix="txn-", dir=transaction_parent)) ) provider_directory = Path(os.path.realpath(tempfile.mkdtemp(prefix=PROVIDER_DIRECTORY_PREFIX))) - mirror_path = provider_directory / WORKSPACE_NAME - mirror_path.mkdir(mode=0o700) os.chmod(transaction_dir, 0o700) # nosemgrep os.chmod(provider_directory, 0o700) # nosemgrep try: - baseline_path = transaction_dir / "baseline" - baseline_path.mkdir(mode=0o700) - mirror = os.fsencode(mirror_path) - baseline_store = os.fsencode(baseline_path) - manifest = make_manifest(root, runtime) - copy_manifest(root, baseline_store, manifest) - copy_manifest(root, mirror, manifest) - transaction_id = str(uuid.uuid4()) - quarantine_root = os.path.join(runtime, b".fixing-quarantine", os.fsencode(transaction_id)) - journal = { - "format": FORMAT_VERSION, - "id": transaction_id, - "state": "mirrored", - "root": os.fsdecode(root), - "runtime": os.fsdecode(runtime), - "metadata_root": os.fsdecode(metadata_root), - "mirror": os.fsdecode(mirror), - "baseline_store": os.fsdecode(baseline_store), - "quarantine_root": os.fsdecode(quarantine_root), - "root_identity": identity(os.lstat(root)), - "runtime_identity": identity(os.lstat(runtime)), - "metadata_root_identity": identity(os.lstat(metadata_root)), - "mirror_identity": identity(os.lstat(mirror)), - "provider_directory_identity": identity(os.lstat(provider_directory)), - "baseline_store_identity": identity(os.lstat(baseline_store)), - "baseline": manifest, - "prepared": None, - "changed": [], - "promoted": [], - "active": None, - "active_started": False, - "evidence": [], - } - journal_path = transaction_dir / "journal.json" - json_dump_atomic(journal_path, journal) + journal_path, journal = _create_mirror_journal( + root, runtime, metadata_root, transaction_dir, provider_directory + ) json_dump_atomic( pointer, { @@ -1279,6 +1292,20 @@ def conflict_message(encoded: str, context: str) -> str: return f"live checkout changed during {context}: {os.fsdecode(decode_path(encoded))}" +def _quarantine_matches_expected( + quarantine: bytes, + encoded: str, + expected: dict[str, Any], + expected_subtree: list[dict[str, Any]] | None, +) -> bool: + if entry_at_absolute(quarantine, encoded) != expected: + return False + return ( + expected_subtree is None + or subtree_manifest_absolute(quarantine, encoded) == expected_subtree + ) + + def quarantine_live_entry( path: Path, journal: dict[str, Any], @@ -1315,12 +1342,7 @@ def quarantine_live_entry( raise TransactionConflict(conflict_message(encoded, context)) from error item["state"] = "quarantined" write_evidence(path, journal) - entry_matches = entry_at_absolute(quarantine, encoded) == expected - subtree_matches = ( - expected_subtree is None - or subtree_manifest_absolute(quarantine, encoded) == expected_subtree - ) - if entry_matches and subtree_matches: + if _quarantine_matches_expected(quarantine, encoded, expected, expected_subtree): return item try: @@ -1981,15 +2003,9 @@ def quarantined_directory( return None -def restore_baseline_entry( - path: Path, - journal: dict[str, Any], - root: bytes, - baseline_store: bytes, - entry: dict[str, Any], - context: str, - subtree: bool = False, -) -> None: +def _recovery_candidate( + journal: dict[str, Any], entry: dict[str, Any], subtree: bool +) -> bytes | None: candidate = quarantined_baseline(journal, entry["path"], entry) if candidate is None and entry["kind"] == "dir" and not subtree: candidate = quarantined_directory(journal, entry["path"]) @@ -1999,38 +2015,43 @@ def restore_baseline_entry( raise TransactionConflict( conflict_message(entry["path"], "recovery directory staging") ) - if candidate is None: - if subtree and entry["kind"] == "dir": - item = stage_directory_subtree( - path, - journal, - root, - baseline_store, - journal["baseline"], - entry, - context, - "recovery", - ) - else: - item = stage_manifest_entry( - path, - journal, - root, - baseline_store, - entry, - context, - "recovery", - ) - else: - item = stage_existing_entry( - path, - journal, - entry["path"], - candidate, - context, - "recovery", - subtree, + return candidate + + +def _stage_recovery_entry( + path: Path, + journal: dict[str, Any], + root: bytes, + baseline_store: bytes, + entry: dict[str, Any], + context: str, + subtree: bool, + candidate: bytes | None, +) -> dict[str, Any]: + if candidate is not None: + return stage_existing_entry( + path, journal, entry["path"], candidate, context, "recovery", subtree ) + if subtree and entry["kind"] == "dir": + return stage_directory_subtree( + path, journal, root, baseline_store, journal["baseline"], entry, context, "recovery" + ) + return stage_manifest_entry(path, journal, root, baseline_store, entry, context, "recovery") + + +def restore_baseline_entry( + path: Path, + journal: dict[str, Any], + root: bytes, + baseline_store: bytes, + entry: dict[str, Any], + context: str, + subtree: bool = False, +) -> None: + candidate = _recovery_candidate(journal, entry, subtree) + item = _stage_recovery_entry( + path, journal, root, baseline_store, entry, context, subtree, candidate + ) expected_subtree = manifest_subtree(journal["baseline"], entry["path"]) if subtree else None install_staged_entry( path, @@ -2338,23 +2359,20 @@ def discard_command(args: argparse.Namespace) -> int: return 0 -def recover_command(args: argparse.Namespace) -> int: - root, runtime, metadata_root, pointer = caller_identity(args) - if not os.path.lexists(pointer): - return 0 - data = pointer_data(pointer) - journal_path = pointer_journal_path(data, root, runtime, metadata_root) - if not os.path.lexists(journal_path): - if data.get("terminal") not in ("committed", "recovered", "discarded"): - raise RuntimeError("nonterminal transaction journal is missing") - if os.path.lexists(journal_path.parent): - owned_private_directory(journal_path.parent, "transaction directory") - remove_tree(journal_path.parent) - unlink_pointer_durable(pointer) - return 0 +def _cleanup_missing_terminal_journal( + pointer: Path, journal_path: Path, data: dict[str, Any] +) -> None: + if data.get("terminal") not in ("committed", "recovered", "discarded"): + raise RuntimeError("nonterminal transaction journal is missing") + if os.path.lexists(journal_path.parent): + owned_private_directory(journal_path.parent, "transaction directory") + remove_tree(journal_path.parent) + unlink_pointer_durable(pointer) + + +def _recover_loaded_journal(args: argparse.Namespace, root: bytes) -> None: path, journal, bound_root, _, mirror, bound_pointer = load_bound_journal( - args, - allow_terminal_cleanup=True, + args, allow_terminal_cleanup=True ) if bound_root != root: raise RuntimeError("transaction recovery root mismatch") @@ -2374,6 +2392,18 @@ def recover_command(args: argparse.Namespace) -> int: journal["evidence"], cleanup_state, ) + + +def recover_command(args: argparse.Namespace) -> int: + root, runtime, metadata_root, pointer = caller_identity(args) + if not os.path.lexists(pointer): + return 0 + data = pointer_data(pointer) + journal_path = pointer_journal_path(data, root, runtime, metadata_root) + if not os.path.lexists(journal_path): + _cleanup_missing_terminal_journal(pointer, journal_path, data) + return 0 + _recover_loaded_journal(args, root) return 0 diff --git a/packages/orchestration/README.md b/packages/orchestration/README.md index 6ae0a50..6ec8290 100644 --- a/packages/orchestration/README.md +++ b/packages/orchestration/README.md @@ -1,11 +1,28 @@ # Phased orchestration -This package implements RAE's staged repository workflow, autonomous executor, +This package implements RAE's graph-native repository workflow, autonomous executor, operator console, artifact contracts, policy validation, and deterministic gates. ## Execution model +New autonomous runs resolve a workflow in this order: explicit `--workflow`, +the locally activated workflow, then the committed +`workflows/graph-native-default.workflow.json`. The resolved workflow, node +guidance, payload contracts, and canonical digest are copied into the run and +remain immutable on resume. + +The default graph includes requirements and design agents, four parallel design +critics, deterministic collection and adjudication, planning, two alignment +extractors, one exclusive build writer, and a five-round repair loop. Read-only +nodes run up to four-wide. Shared command resources serialize, and a writer +drains readers before running alone. Every attempt uses a fresh provider session +and an immutable result envelope. + +The ten ordered stages below are the v1 compatibility engine. Use +`--legacy-linear` only for a temporary new v1 run. Existing v1 request files +select it automatically on resume. + The runtime uses ten ordered stages: | Stage | Responsibility | Primary artifact | @@ -78,8 +95,14 @@ and verification evidence. Useful options: -- `--through ` stops after the selected stage; `--through plan` avoids - repository mutation +- `--workflow ` selects a validated graph-native workflow for a new run +- `--execution-profile ` snapshots an operator-owned mapping from logical + economy, standard, and judgment tiers to Codex model settings; it is mutually + exclusive with global model and reasoning overrides +- `--through ` stops after the selected workflow node +- `--max-concurrency <1..4>` caps concurrent read-only nodes +- `--max-repair-rounds <1..5>` tightens the workflow repair bound +- `--legacy-linear` starts the v1 ten-stage engine - `--checkpoint-policy before-mutation` pauses before the first writable stage - `--checkpoint-policy before-mutation-and-ship` also pauses before the final release decision @@ -89,6 +112,8 @@ Useful options: - `--in-place` uses an explicitly clean target checkout instead of an isolated worktree - `--json` emits machine-readable command output +- `--graph-memory off|read|read-write` controls opt-in local graph retrieval; + the default is `off` and the selected mode is immutable on resume Run `npm run agent -- --help` for the complete option reference. @@ -125,6 +150,49 @@ commands, environment overrides, Git publication, or deployment. See [`operator/README.md`](operator/README.md) for the HTTP and event contract. +## Local graph projections + +Use the umbrella `graph` command to build, inspect, query, explain, or manage +local graph memory: + +```bash +./scripts/rae.sh graph build --project-root /path/to/target-repository +./scripts/rae.sh graph query --project-root /path/to/target-repository \ + --seed 'File:src/main.js' +``` + +Run projections remain under `.pipeline/runs//graph/`. Cross-run +memory remains owner-only under the target repository's Git common directory +at `rae-memory/v1/`. The graph augments context and explanation only. Raw +artifacts, traces, gates, checkpoints, policies, Git state, and human release +decisions remain authoritative. + +Workflow drafts and activation records live under the target Git common +directory at `rae-workflows/v2/`. The owner-only registry uses atomic writes, +an exclusive lock, optimistic revisions, canonical digests, and attributed +activation decisions. The operator editor uses the same registry and rejects +changes while a run is active. + +Workflow schema 2.1 adds bounded map and stream instances, deterministic +transforms, first-success and quorum joins, typed failure collection, +until-dry convergence, and logical execution tiers. Stored 2.0 runs and active +2.0 registry revisions keep their original executor. RAE does not migrate a +private registry automatically. + +Create a draft-only Codex proposal with: + +```bash +./scripts/rae.sh graph workflow propose \ + --project-root /path/to/target-repository \ + --task "Design a bounded topology" \ + --base-workflow graph-native-default \ + --actor "operator-name" \ + --rationale "Draft for review" +``` + +The proposal uses a read-only ephemeral session, validates locally, permits one +correction, and never activates or executes its output. + ## Low-level pipeline API Create local pipeline state: @@ -241,6 +309,16 @@ For changed packages only: ./scripts/verify.sh --changed-only ``` +Run the deterministic workflow-topology fixture separately when scheduler +ordering changes: + +```bash +npm run benchmark:workflow-topology +``` + +It reports fixture event order, critical path, and barrier idle time. It makes +no model-quality or universal speed claim. + The package verifier checks adapters, schemas, stale references, Markdown links, repository hygiene, TypeScript builds, Biome, and Vitest suites. The root repository gate remains: diff --git a/packages/orchestration/contracts/artifacts/execution-trace.schema.json b/packages/orchestration/contracts/artifacts/execution-trace.schema.json index f9c4091..3962e28 100644 --- a/packages/orchestration/contracts/artifacts/execution-trace.schema.json +++ b/packages/orchestration/contracts/artifacts/execution-trace.schema.json @@ -16,9 +16,21 @@ }, "seq": { "type": "integer", "minimum": 1 }, "event_id": { "type": "string", "minLength": 1 }, + "parent_event_id": { "type": "string", "minLength": 1 }, + "input_digests": { + "type": "array", + "items": { "type": "string", "pattern": "^[a-f0-9]{64}$" }, + "uniqueItems": true + }, + "output_digests": { + "type": "array", + "items": { "type": "string", "pattern": "^[a-f0-9]{64}$" }, + "uniqueItems": true + }, "event": { - "type": "string", - "enum": [ + "anyOf": [ + { "type": "string", "pattern": "^workflow_[a-z][a-z0-9_]{1,95}$" }, + { "type": "string", "enum": [ "run_start", "phase_start", "phase_end", @@ -41,27 +53,12 @@ "run_interrupted", "run_blocked", "run_completed" + ] } ] }, "phase": { "type": "string", - "enum": [ - "arm", - "design", - "adversarial-review", - "plan", - "pmatch", - "build", - "quality-static", - "quality-tests", - "denoise", - "quality-frontend", - "quality-backend", - "quality-docs", - "security-review", - "release-readiness", - "post-build" - ] + "pattern": "^[a-z][a-z0-9._-]{0,63}$" }, "status": { "type": "string", diff --git a/packages/orchestration/contracts/graph/graph-context.schema.json b/packages/orchestration/contracts/graph/graph-context.schema.json new file mode 100644 index 0000000..4623795 --- /dev/null +++ b/packages/orchestration/contracts/graph/graph-context.schema.json @@ -0,0 +1,20 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://rae.local/contracts/graph/graph-context.schema.json", + "title": "RAE bounded graph context bundle", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "repository_id", "snapshot_id", "run_id", "phase", "query_id", "seed", "generated_at", "limits", "records"], + "properties": { + "schema_version": { "const": "1.0.0" }, + "repository_id": { "type": "string", "pattern": "^[a-f0-9]{64}$" }, + "snapshot_id": { "type": "string", "pattern": "^[a-f0-9]{64}$" }, + "run_id": { "type": ["string", "null"] }, + "phase": { "type": "string" }, + "query_id": { "type": "string", "pattern": "^[a-f0-9]{64}$" }, + "seed": { "type": "string" }, + "generated_at": { "type": "string", "format": "date-time" }, + "limits": { "type": "object", "additionalProperties": false, "required": ["max_depth", "max_records"], "properties": { "max_depth": { "type": "integer", "maximum": 4 }, "max_records": { "type": "integer", "maximum": 200 } } }, + "records": { "type": "array", "maxItems": 200, "items": { "type": "object", "additionalProperties": false, "required": ["node_id", "kind", "selection_reason", "traversal_path", "trust_class", "source_ref", "source_digest", "staleness", "score", "snippet"], "properties": { "node_id": { "type": "string" }, "kind": { "type": "string" }, "selection_reason": { "type": "string" }, "traversal_path": { "type": "array", "maxItems": 5, "items": { "type": "string" } }, "trust_class": { "type": "string" }, "source_ref": { "type": "string" }, "source_digest": { "type": "string" }, "staleness": { "enum": ["current", "historical", "stale", "conflicting"] }, "score": { "type": "object", "additionalProperties": false, "required": ["exact", "lexical", "distance", "total"], "properties": { "exact": { "type": "number" }, "lexical": { "type": "number" }, "distance": { "type": "number" }, "total": { "type": "number" } } }, "snippet": { "type": "string", "maxLength": 2000 } } } } + } +} diff --git a/packages/orchestration/contracts/graph/graph-edge.schema.json b/packages/orchestration/contracts/graph/graph-edge.schema.json new file mode 100644 index 0000000..2435143 --- /dev/null +++ b/packages/orchestration/contracts/graph/graph-edge.schema.json @@ -0,0 +1,27 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://rae.local/contracts/graph/graph-edge.schema.json", + "title": "RAE graph edge", + "type": "object", + "additionalProperties": false, + "required": ["record_type", "graph_family", "repository_id", "run_id", "kind", "logical_id", "version_id", "from", "to", "source_ref", "source_digest", "projector", "transaction_time", "trust_class", "attributes"], + "properties": { + "record_type": { "const": "edge" }, + "graph_family": { "enum": ["evidence", "workflow", "repository", "memory"] }, + "repository_id": { "type": "string", "pattern": "^[a-f0-9]{64}$" }, + "run_id": { "type": ["string", "null"] }, + "kind": { "enum": ["CONTAINS", "DEPENDS_ON", "REFERENCES", "READS", "WRITES", "DERIVED_FROM", "COVERS", "VERIFIES", "EVALUATES", "AUTHORIZED_BY", "SUPPORTS_CLAIM", "SUPERSEDES", "INVALIDATES", "INSTANCE_OF", "NEXT", "SELECTS", "RECOMMENDS"] }, + "logical_id": { "type": "string", "minLength": 1 }, + "version_id": { "type": "string", "pattern": "^[a-f0-9]{64}$" }, + "from": { "type": "string", "minLength": 1 }, + "to": { "type": "string", "minLength": 1 }, + "source_ref": { "type": "string", "minLength": 1 }, + "source_digest": { "type": "string", "pattern": "^[a-f0-9]{64}$" }, + "projector": { "type": "string", "minLength": 1 }, + "transaction_time": { "type": "string", "format": "date-time" }, + "valid_from": { "type": ["string", "null"], "format": "date-time" }, + "valid_to": { "type": ["string", "null"], "format": "date-time" }, + "trust_class": { "enum": ["authoritative", "verified-derived", "model-proposed", "untrusted"] }, + "attributes": { "type": "object" } + } +} diff --git a/packages/orchestration/contracts/graph/graph-manifest.schema.json b/packages/orchestration/contracts/graph/graph-manifest.schema.json new file mode 100644 index 0000000..b71bdc1 --- /dev/null +++ b/packages/orchestration/contracts/graph/graph-manifest.schema.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://rae.local/contracts/graph/graph-manifest.schema.json", + "title": "RAE graph manifest", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "projector", "repository_id", "snapshot_id", "run_id", "transaction_time", "node_count", "edge_count", "nodes_digest", "edges_digest", "canonical_digest", "limits", "validation"], + "properties": { + "schema_version": { "const": "1.0.0" }, + "projector": { "type": "string" }, + "repository_id": { "type": "string", "pattern": "^[a-f0-9]{64}$" }, + "snapshot_id": { "type": "string", "pattern": "^[a-f0-9]{64}$" }, + "run_id": { "type": ["string", "null"] }, + "transaction_time": { "type": "string", "format": "date-time" }, + "node_count": { "type": "integer", "minimum": 0, "maximum": 250000 }, + "edge_count": { "type": "integer", "minimum": 0, "maximum": 1000000 }, + "nodes_digest": { "type": "string", "pattern": "^[a-f0-9]{64}$" }, + "edges_digest": { "type": "string", "pattern": "^[a-f0-9]{64}$" }, + "canonical_digest": { "type": "string", "pattern": "^[a-f0-9]{64}$" }, + "limits": { "type": "object", "additionalProperties": false, "required": ["max_nodes", "max_edges", "max_file_bytes"], "properties": { "max_nodes": { "const": 250000 }, "max_edges": { "const": 1000000 }, "max_file_bytes": { "const": 1048576 } } }, + "validation": { "type": "object", "additionalProperties": false, "required": ["valid", "issues"], "properties": { "valid": { "type": "boolean" }, "issues": { "type": "array", "items": { "type": "string" } } } } + } +} diff --git a/packages/orchestration/contracts/graph/graph-node.schema.json b/packages/orchestration/contracts/graph/graph-node.schema.json new file mode 100644 index 0000000..cc5a330 --- /dev/null +++ b/packages/orchestration/contracts/graph/graph-node.schema.json @@ -0,0 +1,25 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://rae.local/contracts/graph/graph-node.schema.json", + "title": "RAE graph node", + "type": "object", + "additionalProperties": false, + "required": ["record_type", "graph_family", "repository_id", "run_id", "kind", "logical_id", "version_id", "source_ref", "source_digest", "projector", "transaction_time", "trust_class", "attributes"], + "properties": { + "record_type": { "const": "node" }, + "graph_family": { "enum": ["evidence", "workflow", "repository", "memory"] }, + "repository_id": { "type": "string", "pattern": "^[a-f0-9]{64}$" }, + "run_id": { "type": ["string", "null"] }, + "kind": { "enum": ["Repository", "ProjectSnapshot", "Run", "PhaseAttempt", "WorkflowRevision", "AgentNode", "NodeAttempt", "Join", "LoopIteration", "Candidate", "Evaluation", "Recommendation", "ActivationDecision", "Requirement", "Constraint", "PlanTask", "TestCase", "File", "ArtifactVersion", "CommandExecution", "GateDecision", "CheckpointDecision", "Finding", "Claim", "SourceDocument"] }, + "logical_id": { "type": "string", "minLength": 1 }, + "version_id": { "type": "string", "pattern": "^[a-f0-9]{64}$" }, + "source_ref": { "type": "string", "minLength": 1 }, + "source_digest": { "type": "string", "pattern": "^[a-f0-9]{64}$" }, + "projector": { "type": "string", "minLength": 1 }, + "transaction_time": { "type": "string", "format": "date-time" }, + "valid_from": { "type": ["string", "null"], "format": "date-time" }, + "valid_to": { "type": ["string", "null"], "format": "date-time" }, + "trust_class": { "enum": ["authoritative", "verified-derived", "model-proposed", "untrusted"] }, + "attributes": { "type": "object" } + } +} diff --git a/packages/orchestration/contracts/graph/memory-decision.schema.json b/packages/orchestration/contracts/graph/memory-decision.schema.json new file mode 100644 index 0000000..fc43769 --- /dev/null +++ b/packages/orchestration/contracts/graph/memory-decision.schema.json @@ -0,0 +1,19 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://rae.local/contracts/graph/memory-decision.schema.json", + "title": "RAE graph memory decision", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "decision_id", "candidate_id", "decision", "actor", "rationale", "source_ref", "source_digest", "recorded_at"], + "properties": { + "schema_version": { "const": "1.0.0" }, + "decision_id": { "type": "string", "pattern": "^[a-f0-9]{64}$" }, + "candidate_id": { "type": "string", "minLength": 1 }, + "decision": { "enum": ["promoted", "rejected", "superseded", "invalidated"] }, + "actor": { "type": "string", "minLength": 1 }, + "rationale": { "type": "string", "minLength": 1 }, + "source_ref": { "type": "string", "minLength": 1 }, + "source_digest": { "type": "string", "pattern": "^[a-f0-9]{64}$" }, + "recorded_at": { "type": "string", "format": "date-time" } + } +} diff --git a/packages/orchestration/contracts/workflows/execution-profile-v1.schema.json b/packages/orchestration/contracts/workflows/execution-profile-v1.schema.json new file mode 100644 index 0000000..a935d8f --- /dev/null +++ b/packages/orchestration/contracts/workflows/execution-profile-v1.schema.json @@ -0,0 +1,30 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://rae.local/contracts/workflows/execution-profile-v1.schema.json", + "title": "RAE operator-owned Codex execution profile", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "profile_id", "tiers"], + "properties": { + "schema_version": { "const": "1.0.0" }, + "profile_id": { "type": "string", "pattern": "^[a-z][a-z0-9-]{2,63}$" }, + "tiers": { + "type": "object", "additionalProperties": false, + "required": ["economy", "standard", "judgment"], + "properties": { + "economy": { "$ref": "#/$defs/mapping" }, + "standard": { "$ref": "#/$defs/mapping" }, + "judgment": { "$ref": "#/$defs/mapping" } + } + } + }, + "$defs": { + "mapping": { + "type": "object", "additionalProperties": false, "required": ["model", "reasoning_effort"], + "properties": { + "model": { "type": "string", "minLength": 1, "maxLength": 128 }, + "reasoning_effort": { "enum": ["low", "medium", "high", "xhigh"] } + } + } + } +} diff --git a/packages/orchestration/contracts/workflows/node-envelope-v2.1.schema.json b/packages/orchestration/contracts/workflows/node-envelope-v2.1.schema.json new file mode 100644 index 0000000..09a02f4 --- /dev/null +++ b/packages/orchestration/contracts/workflows/node-envelope-v2.1.schema.json @@ -0,0 +1,35 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://rae.local/contracts/workflows/node-envelope-v2.1.schema.json", + "title": "Immutable workflow node-instance result envelope v2.1", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "run_id", "workflow_digest", "node_id", "instance_id", "attempt", "status", "payload", "findings", "evidence_refs", "ownership", "changed_paths", "command_evidence", "resource_usage", "input_digest", "output_digest", "execution_tier"], + "properties": { + "schema_version": { "const": "2.1.0" }, + "run_id": { "type": "string", "minLength": 1 }, + "workflow_digest": { "type": "string", "pattern": "^[a-f0-9]{64}$" }, + "node_id": { "type": "string", "minLength": 1 }, + "instance_id": { "type": "string", "pattern": "^[a-zA-Z0-9._:-]{1,192}$" }, + "parent_node": { "type": ["string", "null"] }, + "item_key": { "type": ["string", "null"], "maxLength": 512 }, + "item_digest": { "type": ["string", "null"], "pattern": "^(?:[a-f0-9]{64})?$" }, + "attempt": { "type": "integer", "minimum": 1, "maximum": 3 }, + "loop_iteration": { "type": "integer", "minimum": 1, "maximum": 5 }, + "status": { "enum": ["passed", "failed", "blocked", "stopped", "skipped", "collected"] }, + "failure": { "type": ["object", "null"] }, + "payload": {}, + "findings": { "type": "array", "maxItems": 1024, "items": { "type": "object" } }, + "evidence_refs": { "type": "array", "maxItems": 1024, "items": { "type": "string" } }, + "ownership": { "type": "object" }, + "changed_paths": { "type": "array", "maxItems": 4096, "uniqueItems": true, "items": { "type": "string" } }, + "command_evidence": { "type": "array", "maxItems": 1024, "items": { "type": "object" } }, + "resource_usage": { "type": "object" }, + "input_digest": { "type": "string", "pattern": "^[a-f0-9]{64}$" }, + "output_digest": { "type": "string", "pattern": "^[a-f0-9]{64}$" }, + "execution_tier": { "enum": ["economy", "standard", "judgment", "runtime"] }, + "selection": { "type": ["object", "null"] }, + "quorum": { "type": ["object", "null"] }, + "convergence": { "type": ["object", "null"] } + } +} diff --git a/packages/orchestration/contracts/workflows/node-envelope-v2.schema.json b/packages/orchestration/contracts/workflows/node-envelope-v2.schema.json new file mode 100644 index 0000000..c56e1a2 --- /dev/null +++ b/packages/orchestration/contracts/workflows/node-envelope-v2.schema.json @@ -0,0 +1,26 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://rae.local/contracts/workflows/node-envelope-v2.schema.json", + "title": "Immutable workflow node result envelope", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "run_id", "workflow_digest", "node_id", "attempt", "status", "payload", "findings", "evidence_refs", "ownership", "changed_paths", "command_evidence", "resource_usage", "input_digest", "output_digest"], + "properties": { + "schema_version": { "const": "2.0.0" }, + "run_id": { "type": "string", "minLength": 1 }, + "workflow_digest": { "type": "string", "pattern": "^[a-f0-9]{64}$" }, + "node_id": { "type": "string", "minLength": 1 }, + "attempt": { "type": "integer", "minimum": 1, "maximum": 3 }, + "loop_iteration": { "type": "integer", "minimum": 1, "maximum": 5 }, + "status": { "enum": ["passed", "failed", "blocked", "stopped", "skipped"] }, + "payload": {}, + "findings": { "type": "array", "maxItems": 1024, "items": { "type": "object" } }, + "evidence_refs": { "type": "array", "maxItems": 1024, "items": { "type": "string" } }, + "ownership": { "type": "object" }, + "changed_paths": { "type": "array", "maxItems": 4096, "uniqueItems": true, "items": { "type": "string" } }, + "command_evidence": { "type": "array", "maxItems": 1024, "items": { "type": "object" } }, + "resource_usage": { "type": "object" }, + "input_digest": { "type": "string", "pattern": "^[a-f0-9]{64}$" }, + "output_digest": { "type": "string", "pattern": "^[a-f0-9]{64}$" } + } +} diff --git a/packages/orchestration/contracts/workflows/workflow-v2.1.schema.json b/packages/orchestration/contracts/workflows/workflow-v2.1.schema.json new file mode 100644 index 0000000..d13094d --- /dev/null +++ b/packages/orchestration/contracts/workflows/workflow-v2.1.schema.json @@ -0,0 +1,122 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://rae.local/contracts/workflows/workflow-v2.1.schema.json", + "title": "RAE graph-native workflow v2.1", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "workflow_id", "revision", "entry_node", "terminal_node", "nodes", "edges"], + "properties": { + "schema_version": { "const": "2.1.0" }, + "workflow_id": { "type": "string", "pattern": "^[a-z][a-z0-9-]{2,63}$" }, + "revision": { "type": "integer", "minimum": 1 }, + "title": { "type": "string", "minLength": 1, "maxLength": 160 }, + "entry_node": { "$ref": "#/$defs/nodeId" }, + "terminal_node": { "$ref": "#/$defs/nodeId" }, + "nodes": { "type": "array", "minItems": 2, "maxItems": 64, "items": { "$ref": "#/$defs/node" } }, + "edges": { "type": "array", "minItems": 1, "maxItems": 256, "items": { "$ref": "#/$defs/edge" } }, + "payload_contracts": { "type": "object", "maxProperties": 64, "additionalProperties": { "type": "object" } }, + "budgets": { + "type": "object", "additionalProperties": false, + "properties": { + "max_concurrency": { "type": "integer", "minimum": 1, "maximum": 4 }, + "max_repair_rounds": { "type": "integer", "minimum": 0, "maximum": 5 }, + "max_attempts_per_node": { "type": "integer", "minimum": 1, "maximum": 3 }, + "max_dynamic_instances": { "type": "integer", "minimum": 1, "maximum": 128 }, + "max_pipeline_depth": { "type": "integer", "minimum": 1, "maximum": 4 }, + "max_map_items": { "type": "integer", "minimum": 1, "maximum": 32 } + } + } + }, + "$defs": { + "nodeId": { "type": "string", "pattern": "^[a-z][a-z0-9._-]{0,63}$" }, + "pointer": { "type": "string", "pattern": "^(|/(?:[^~/]|~[01])*)$", "maxLength": 512 }, + "failure": { + "type": "object", "additionalProperties": false, "required": ["mode"], + "properties": { + "mode": { "enum": ["fail-workflow", "collect"] }, + "max_failures": { "type": "integer", "minimum": 0, "maximum": 31 }, + "minimum_successes": { "type": "integer", "minimum": 1, "maximum": 32 } + } + }, + "map": { + "type": "object", "additionalProperties": false, + "required": ["source_pointer", "stable_key_pointer"], + "properties": { + "source_pointer": { "$ref": "#/$defs/pointer" }, + "stable_key_pointer": { "$ref": "#/$defs/pointer" }, + "max_items": { "type": "integer", "minimum": 1, "maximum": 32 } + } + }, + "transform": { + "type": "object", "additionalProperties": false, "required": ["operation"], + "properties": { + "operation": { "enum": ["select", "flatten", "deduplicate", "sort", "limit", "group", "cartesian"] }, + "source_pointer": { "$ref": "#/$defs/pointer" }, + "value_pointer": { "$ref": "#/$defs/pointer" }, + "key_pointer": { "$ref": "#/$defs/pointer" }, + "pointers": { "type": "array", "minItems": 2, "maxItems": 4, "uniqueItems": true, "items": { "$ref": "#/$defs/pointer" } }, + "limit": { "type": "integer", "minimum": 1, "maximum": 128 }, + "descending": { "type": "boolean" } + } + }, + "quorum": { + "type": "object", "additionalProperties": false, "required": ["threshold"], + "properties": { + "threshold": { "type": "integer", "minimum": 1, "maximum": 64 }, + "groups": { + "type": "array", "minItems": 1, "maxItems": 16, + "items": { + "type": "object", "additionalProperties": false, "required": ["id", "members", "threshold"], + "properties": { + "id": { "type": "string", "pattern": "^[a-z][a-z0-9-]{0,31}$" }, + "members": { "type": "array", "minItems": 1, "maxItems": 32, "uniqueItems": true, "items": { "$ref": "#/$defs/nodeId" } }, + "threshold": { "type": "integer", "minimum": 1, "maximum": 32 } + } + } + } + } + }, + "loop": { + "type": "object", "additionalProperties": false, "required": ["max_iterations", "members"], + "properties": { + "mode": { "enum": ["bounded", "until-dry"] }, + "max_iterations": { "type": "integer", "minimum": 1, "maximum": 5 }, + "members": { "type": "array", "minItems": 1, "maxItems": 32, "uniqueItems": true, "items": { "$ref": "#/$defs/nodeId" } }, + "source_pointer": { "$ref": "#/$defs/pointer" }, + "stable_key_pointer": { "$ref": "#/$defs/pointer" } + } + }, + "node": { + "type": "object", "additionalProperties": false, "required": ["id", "kind", "access", "guidance"], + "properties": { + "id": { "$ref": "#/$defs/nodeId" }, + "kind": { "enum": ["agent", "map", "transform", "join", "gate", "checkpoint", "loop", "terminal"] }, + "access": { "enum": ["read", "write", "control"] }, + "guidance": { "type": "string", "minLength": 1, "maxLength": 12000 }, + "role": { "type": "string", "maxLength": 128 }, + "payload_contract": { "type": "string", "pattern": "^[A-Za-z][A-Za-z0-9._-]{0,63}$" }, + "tier": { "enum": ["economy", "standard", "judgment"] }, + "join": { "enum": ["all", "any", "quorum"] }, + "quorum": { "$ref": "#/$defs/quorum" }, + "map": { "$ref": "#/$defs/map" }, + "transform": { "$ref": "#/$defs/transform" }, + "failure_handling": { "$ref": "#/$defs/failure" }, + "resource": { "type": "string", "maxLength": 128 }, + "ownership_plan": { "type": "boolean" }, + "mutation_checkpoint": { "type": "boolean" }, + "verification": { "type": "boolean" }, + "loop": { "$ref": "#/$defs/loop" } + } + }, + "edge": { + "type": "object", "additionalProperties": false, "required": ["from", "to", "type"], + "properties": { + "from": { "$ref": "#/$defs/nodeId" }, + "to": { "$ref": "#/$defs/nodeId" }, + "type": { "enum": ["sequence", "artifact", "stream", "condition", "loop-back"] }, + "artifact": { "type": "string", "maxLength": 128 }, + "condition": { "enum": ["success", "failure", "blocking-findings", "budget-available"] } + } + } + } +} diff --git a/packages/orchestration/contracts/workflows/workflow-v2.schema.json b/packages/orchestration/contracts/workflows/workflow-v2.schema.json new file mode 100644 index 0000000..7a50ca6 --- /dev/null +++ b/packages/orchestration/contracts/workflows/workflow-v2.schema.json @@ -0,0 +1,75 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://rae.local/contracts/workflows/workflow-v2.schema.json", + "title": "RAE graph-native workflow", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "workflow_id", "revision", "entry_node", "terminal_node", "nodes", "edges"], + "properties": { + "schema_version": { "const": "2.0.0" }, + "workflow_id": { "type": "string", "pattern": "^[a-z][a-z0-9-]{2,63}$" }, + "revision": { "type": "integer", "minimum": 1 }, + "title": { "type": "string", "minLength": 1, "maxLength": 160 }, + "entry_node": { "$ref": "#/$defs/nodeId" }, + "terminal_node": { "$ref": "#/$defs/nodeId" }, + "nodes": { + "type": "array", "minItems": 2, "maxItems": 64, + "items": { "$ref": "#/$defs/node" } + }, + "edges": { + "type": "array", "minItems": 1, "maxItems": 256, + "items": { "$ref": "#/$defs/edge" } + }, + "payload_contracts": { + "type": "object", "maxProperties": 64, + "additionalProperties": { "type": "object" } + }, + "budgets": { + "type": "object", "additionalProperties": false, + "properties": { + "max_concurrency": { "type": "integer", "minimum": 1, "maximum": 4 }, + "max_repair_rounds": { "type": "integer", "minimum": 0, "maximum": 5 }, + "max_attempts_per_node": { "type": "integer", "minimum": 1, "maximum": 3 } + } + } + }, + "$defs": { + "nodeId": { "type": "string", "pattern": "^[a-z][a-z0-9._-]{0,63}$" }, + "node": { + "type": "object", "additionalProperties": false, + "required": ["id", "kind", "access", "guidance"], + "properties": { + "id": { "$ref": "#/$defs/nodeId" }, + "kind": { "enum": ["agent", "join", "gate", "checkpoint", "loop", "terminal"] }, + "access": { "enum": ["read", "write", "control"] }, + "guidance": { "type": "string", "minLength": 1, "maxLength": 12000 }, + "role": { "type": "string", "maxLength": 128 }, + "payload_contract": { "type": "string", "pattern": "^[A-Za-z][A-Za-z0-9._-]{0,63}$" }, + "join": { "enum": ["all", "any"] }, + "resource": { "type": "string", "maxLength": 128 }, + "ownership_plan": { "type": "boolean" }, + "mutation_checkpoint": { "type": "boolean" }, + "verification": { "type": "boolean" }, + "loop": { + "type": "object", "additionalProperties": false, + "required": ["max_iterations", "members"], + "properties": { + "max_iterations": { "type": "integer", "minimum": 1, "maximum": 5 }, + "members": { "type": "array", "minItems": 1, "maxItems": 32, "uniqueItems": true, "items": { "$ref": "#/$defs/nodeId" } } + } + } + } + }, + "edge": { + "type": "object", "additionalProperties": false, + "required": ["from", "to", "type"], + "properties": { + "from": { "$ref": "#/$defs/nodeId" }, + "to": { "$ref": "#/$defs/nodeId" }, + "type": { "enum": ["sequence", "artifact", "condition", "loop-back"] }, + "artifact": { "type": "string", "maxLength": 128 }, + "condition": { "enum": ["success", "failure", "blocking-findings", "budget-available"] } + } + } + } +} diff --git a/packages/orchestration/operator/README.md b/packages/orchestration/operator/README.md index a2aa60d..67e82b0 100644 --- a/packages/orchestration/operator/README.md +++ b/packages/orchestration/operator/README.md @@ -59,6 +59,18 @@ overrides, raw trace access, forced cleanup, commit, push, or publish controls. Cleanup delegates to the pipeline's ownership- and dirty-state-validating worktree cleanup operation. +The workflow editor lists immutable revisions, renders synchronized SVG and +structured node/edge views, validates drafts, compares revisions, displays +budgets and activation history, and activates only after exact digest +confirmation. Native forms and the structured list provide every authoring +operation; canvas dragging is not required. Registry mutations are rejected +while any allowlisted project run is active. + +Run projections include bounded graph health counts when a projection exists: +availability, validation state, node and edge counts, stale-source count, and +stale-memory and unresolved-conflict counts. The API does not expose raw graph records, absolute +paths, prompts, provider metadata, or untrusted memory text. + Only one process started by a server instance may be active at once. Interrupt signals that owned process group, records `interrupted` after it exits, and removes an autonomous lock only when its recorded PID matches the owned child. diff --git a/packages/orchestration/operator/demo/README.md b/packages/orchestration/operator/demo/README.md new file mode 100644 index 0000000..a089f70 --- /dev/null +++ b/packages/orchestration/operator/demo/README.md @@ -0,0 +1,18 @@ +# Static Evidence Dossier demo + +The GitHub Pages demo is derived from the maintained operator console at build +time. It copies the canonical HTML, CSS, and JavaScript modules, then adds only +this directory's simulation notice, sanitized fixture adapter, and visible +action labels. + +The demo runs no command, contacts no operator API, and stores no run state. +Every interaction changes only the in-memory fixture until the page is reloaded. +It is a product walkthrough, not evidence from an autonomous run. + +Build it from the repository root with: + +```bash +npm --prefix packages/orchestration run build:pages-demo +``` + +The generated site is written to the ignored `dist/pages-demo/` directory. diff --git a/packages/orchestration/operator/demo/demo.css.txt b/packages/orchestration/operator/demo/demo.css.txt new file mode 100644 index 0000000..2dfe823 --- /dev/null +++ b/packages/orchestration/operator/demo/demo.css.txt @@ -0,0 +1,96 @@ +/* Keeps the static simulation unmistakable while preserving the canonical Evidence Dossier UI. */ + +:root { + --demo-notice-height: 2.75rem; +} + +.demo-notice { + position: sticky; + top: 0; + z-index: 50; + min-height: var(--demo-notice-height); + display: flex; + align-items: center; + justify-content: center; + gap: 0.65rem; + padding: 0.55rem 1rem; + border-bottom: 1px solid var(--ink); + background: var(--trace); + color: var(--sheet); + font-size: 0.76rem; + line-height: 1.25; + text-align: center; +} + +.demo-notice strong { + font-family: var(--mono); + font-size: 0.7rem; + letter-spacing: 0.08em; + text-transform: uppercase; +} + +.demo-notice a { + color: inherit; + text-underline-offset: 0.18em; +} + +.head { + top: var(--demo-notice-height); +} + +.skip { + position: fixed; + top: 0.5rem; + left: 0.5rem; + z-index: 60; + padding: 0.45rem 0.65rem; + background: var(--sheet); + color: var(--ink); + transform: translateY(-200%); +} + +.skip:focus { + transform: translateY(0); +} + +.simulated-label { + display: inline-block; + margin-left: 0.45rem; + padding-left: 0.45rem; + border-left: 1px solid currentColor; + font: 600 0.58rem/1 var(--mono); + letter-spacing: 0.04em; + text-transform: uppercase; + opacity: 0.78; +} + +.decision-action .simulated-label, +.run-controls .simulated-label { + font-size: 0.55rem; +} + +.primary-action .simulated-label { + display: inline-block; +} + +@media (max-width: 640px) { + .demo-notice { + position: static; + justify-content: flex-start; + flex-wrap: wrap; + text-align: left; + } + + .head { + top: 0; + } + + .head__sep, + .head__surface { + display: none; + } + + .demo-notice a { + width: 100%; + } +} diff --git a/packages/orchestration/operator/demo/mock-api.js b/packages/orchestration/operator/demo/mock-api.js new file mode 100644 index 0000000..8291cf5 --- /dev/null +++ b/packages/orchestration/operator/demo/mock-api.js @@ -0,0 +1,420 @@ +/** Supplies sanitized, in-memory API responses for the static operator-console demonstration. */ + +const PHASES = [ + "arm", + "design", + "adversarial-review", + "plan", + "pmatch", + "build", + "quality-static", + "quality-tests", + "post-build", + "release-readiness", +]; + +const SIMULATED_ACTION_IDS = [ + "new-run-button", + "start-submit", + "stop-button", + "interrupt-button", + "resume-button", + "cleanup-button", + "confirm-submit", +]; + +const baseRun = { + phase_order: PHASES, + workspace_mode: "worktree", + evidence: { present: 6 }, + resources: { agent_calls: 11, input: 184220, output: 28310, cost: null }, + graph_health: { + available: false, + valid: false, + node_count: 0, + edge_count: 0, + stale_sources: 0, + stale_memory: 0, + unresolved_conflicts: 0, + }, +}; + +const runs = [ + { + ...baseRun, + id: "run-7f3a2c91", + task: "Add a tested health endpoint and document the public behavior.", + branch: "pipeline/run-7f3a2c91", + workspace_label: ".git/rae-worktrees/run-7f3a2c91", + status: "awaiting", + current_phase: "build", + started_at: "2026-07-23T14:02:00.000Z", + updated_at: "2026-07-23T14:08:00.000Z", + completed_gates: [ + "arm-gate", + "design-gate", + "adversarial-review-gate", + "plan-gate", + "pmatch-gate", + ], + gates: [ + { gate_id: "arm-gate", phase: "arm", status: "pass", artifact_ref: "brief · a91f" }, + { gate_id: "design-gate", phase: "design", status: "pass", artifact_ref: "design · 3c20" }, + { + gate_id: "adversarial-review-gate", + phase: "adversarial-review", + status: "pass", + artifact_ref: "review · 88e1", + }, + { gate_id: "plan-gate", phase: "plan", status: "pass", artifact_ref: "plan · b7d4" }, + { gate_id: "pmatch-gate", phase: "pmatch", status: "pass", artifact_ref: "drift · 0f2a" }, + { + gate_id: "build-gate", + phase: "build", + status: "pending", + artifact_ref: "build · 7c…e19", + }, + ], + checkpoints: [ + { + checkpoint_id: "cp-4b91-build", + purpose: "mutation", + phase: "build", + status: "pending", + message: + "Plan-owned implementation is staged. Gate policy before-mutation-and-ship requires an operator record before quality-static runs.", + requested_at: "2026-07-23T14:08:00.000Z", + }, + ], + controls: { stop: true, interrupt: true, resume: false, cleanup: false }, + }, + { + ...baseRun, + id: "run-91bc08d2", + task: "Harden report path confinement for Ralph fixing transactions.", + branch: "pipeline/run-91bc08d2", + workspace_label: ".git/rae-worktrees/run-91bc08d2", + status: "completed", + current_phase: "release-readiness", + started_at: "2026-07-23T12:42:00.000Z", + updated_at: "2026-07-23T13:12:00.000Z", + completed_gates: PHASES.map((phase) => `${phase}-gate`), + gates: PHASES.map((phase, index) => ({ + gate_id: `${phase}-gate`, + phase, + status: "pass", + artifact_ref: `evidence · ${String(index + 1).padStart(2, "0")}`, + })), + checkpoints: [], + controls: { stop: false, interrupt: false, resume: false, cleanup: true }, + }, + { + ...baseRun, + id: "run-2e11d4a0", + task: "Correct a scoped documentation claim without widening the change set.", + branch: "pipeline/run-2e11d4a0", + workspace_label: ".git/rae-worktrees/run-2e11d4a0", + status: "blocked", + current_phase: "pmatch", + started_at: "2026-07-23T11:20:00.000Z", + updated_at: "2026-07-23T11:58:00.000Z", + completed_gates: ["arm-gate", "design-gate", "adversarial-review-gate", "plan-gate"], + gates: [ + { gate_id: "arm-gate", phase: "arm", status: "pass", artifact_ref: "brief · d102" }, + { gate_id: "design-gate", phase: "design", status: "pass", artifact_ref: "design · 73a4" }, + { + gate_id: "adversarial-review-gate", + phase: "adversarial-review", + status: "pass", + artifact_ref: "review · 220c", + }, + { gate_id: "plan-gate", phase: "plan", status: "pass", artifact_ref: "plan · c814" }, + { gate_id: "pmatch-gate", phase: "pmatch", status: "failed", artifact_ref: "drift · 91ff" }, + ], + checkpoints: [], + controls: { stop: false, interrupt: false, resume: true, cleanup: true }, + }, +]; + +const eventsByRun = new Map( + Object.entries({ + "run-7f3a2c91": [ + { + seq: 1, + ts: "2026-07-23T14:02:05.000Z", + phase: "arm", + event: "artifact_recorded", + artifact_ref: "brief · a91f", + status: "pass", + tier: "local", + }, + { + seq: 2, + ts: "2026-07-23T14:03:18.000Z", + phase: "design", + event: "gate_completed", + gate_id: "design-gate", + status: "pass", + tier: "local", + }, + { + seq: 3, + ts: "2026-07-23T14:04:42.000Z", + phase: "adversarial-review", + event: "review_completed", + artifact_ref: "review · 88e1", + status: "pass", + tier: "local", + }, + { + seq: 4, + ts: "2026-07-23T14:06:09.000Z", + phase: "plan", + event: "plan_validated", + artifact_ref: "plan · b7d4", + status: "pass", + tier: "local", + }, + { + seq: 5, + ts: "2026-07-23T14:07:31.000Z", + phase: "pmatch", + event: "drift_check_completed", + gate_id: "pmatch-gate", + status: "pass", + tier: "local", + }, + { + seq: 6, + ts: "2026-07-23T14:08:00.000Z", + phase: "build", + event: "checkpoint_requested", + event_id: "cp-4b91-build", + status: "pending", + tier: "human", + }, + ], + "run-91bc08d2": [ + { + seq: 1, + ts: "2026-07-23T12:42:03.000Z", + phase: "arm", + event: "run_started", + event_id: "evt-01", + status: "pass", + tier: "local", + }, + { + seq: 2, + ts: "2026-07-23T13:12:00.000Z", + phase: "release-readiness", + event: "release_gate_completed", + gate_id: "release-readiness-gate", + status: "pass", + tier: "local", + }, + ], + "run-2e11d4a0": [ + { + seq: 1, + ts: "2026-07-23T11:20:02.000Z", + phase: "arm", + event: "run_started", + event_id: "evt-01", + status: "pass", + tier: "local", + }, + { + seq: 2, + ts: "2026-07-23T11:58:00.000Z", + phase: "pmatch", + event: "drift_detected", + gate_id: "pmatch-gate", + status: "failed", + tier: "local", + }, + ], + }), +); + +function json(payload, status = 200) { + return new Response(JSON.stringify(payload), { + status, + headers: { "content-type": "application/json; charset=utf-8" }, + }); +} + +function bodyOf(options) { + return options.body ? JSON.parse(options.body) : {}; +} + +function updateRun(run, changes) { + Object.assign(run, changes, { updated_at: "2026-07-23T14:24:00.000Z" }); +} + +function applyCheckpointDecision(run, body) { + const checkpoint = run.checkpoints.find((item) => item.checkpoint_id === body.checkpoint_id); + if (!checkpoint) return json({ error: { message: "Fixture checkpoint not found." } }, 404); + checkpoint.status = body.decision; + if (body.decision === "approve") { + const buildGate = run.gates.find((gate) => gate.gate_id === "build-gate"); + buildGate.status = "pass"; + if (!run.completed_gates.includes("build-gate")) run.completed_gates.push("build-gate"); + updateRun(run, { status: "running", current_phase: "quality-static" }); + } else { + updateRun(run, { status: body.decision === "reject" ? "blocked" : "awaiting" }); + } + const events = eventsByRun.get(run.id); + events.push({ + seq: events.length + 1, + ts: run.updated_at, + phase: checkpoint.phase, + event: `checkpoint_${body.decision}`, + event_id: checkpoint.checkpoint_id, + status: body.decision, + tier: "human", + }); + return json({ ok: true, simulated: true }); +} + +function createRun(body) { + const id = `run-demo-${String(runs.length + 1).padStart(2, "0")}`; + const run = { + ...structuredClone(baseRun), + id, + task: body.task, + branch: `pipeline/${id}`, + workspace_label: `.git/rae-worktrees/${id}`, + status: "awaiting", + current_phase: "arm", + started_at: "2026-07-23T14:24:00.000Z", + updated_at: "2026-07-23T14:24:00.000Z", + completed_gates: [], + gates: [ + { gate_id: "arm-gate", phase: "arm", status: "pending", artifact_ref: "brief · fixture" }, + ], + checkpoints: [ + { + checkpoint_id: `${id}-arm`, + purpose: "mutation", + phase: "arm", + status: "pending", + message: "This simulated run is waiting at its first fixture checkpoint.", + requested_at: "2026-07-23T14:24:00.000Z", + }, + ], + controls: { stop: true, interrupt: true, resume: false, cleanup: false }, + }; + runs.unshift(run); + eventsByRun.set(id, [ + { + seq: 1, + ts: run.started_at, + phase: "arm", + event: "fixture_run_created", + event_id: `${id}-created`, + status: "pending", + tier: "simulation", + }, + ]); + return json({ run_id: id, simulated: true }, 202); +} + +function streamResponse(signal) { + const stream = new ReadableStream({ + start(controller) { + if (signal?.aborted) { + controller.close(); + return; + } + signal?.addEventListener("abort", () => controller.close(), { once: true }); + }, + }); + return new Response(stream, { + status: 200, + headers: { "content-type": "application/x-ndjson" }, + }); +} + +function getResponse(path, options) { + if (path === "/projects") { + return json({ + projects: [{ id: "project_fixture", label: "sebastianspicker/rae · fixture" }], + }); + } + if (path.endsWith("/events/stream")) return streamResponse(options.signal); + + const eventsMatch = path.match(/^\/projects\/[^/]+\/runs\/([^/]+)\/events$/); + if (eventsMatch) { + const events = structuredClone(eventsByRun.get(decodeURIComponent(eventsMatch[1])) || []); + return json({ events, next_after: events.at(-1)?.seq || 0 }); + } + + if (/^\/projects\/[^/]+\/runs$/.test(path)) return json({ runs: structuredClone(runs) }); + return null; +} + +function actionStatus(action) { + if (action === "stop") return "stopping"; + if (action === "resume") return "running"; + return "interrupted"; +} + +function postAction(actionMatch, options) { + const runId = decodeURIComponent(actionMatch[1]); + const action = actionMatch[2]; + const run = runs.find((item) => item.id === runId); + if (!run) return json({ error: { message: "Fixture run not found." } }, 404); + if (action === "checkpoint-decision") return applyCheckpointDecision(run, bodyOf(options)); + if (action === "cleanup") { + runs.splice(runs.indexOf(run), 1); + return json({ ok: true, simulated: true }); + } + updateRun(run, { status: actionStatus(action) }); + if (action === "interrupt") { + run.controls = { stop: false, interrupt: false, resume: true, cleanup: true }; + } + return json({ ok: true, simulated: true }); +} + +function postResponse(path, options) { + if (/^\/projects\/[^/]+\/runs$/.test(path)) return createRun(bodyOf(options)); + + const actionMatch = path.match( + /^\/projects\/[^/]+\/runs\/([^/]+)\/(stop|resume|interrupt|cleanup|checkpoint-decision)$/, + ); + return actionMatch ? postAction(actionMatch, options) : null; +} + +function demoFetch(input, options = {}) { + const url = new URL(typeof input === "string" ? input : input.url, location.href); + if (!url.pathname.startsWith("/api/v1/")) { + throw new Error("The static simulation does not permit network requests."); + } + + const path = url.pathname.slice("/api/v1".length); + const method = String(options.method || "GET").toUpperCase(); + const response = method === "GET" ? getResponse(path, options) : postResponse(path, options); + return response || json({ error: { message: "Unsupported static-demo request." } }, 404); +} + +function markSimulatedControls() { + const controls = [ + ...SIMULATED_ACTION_IDS.map((id) => document.getElementById(id)), + ...document.querySelectorAll("[data-decision]"), + ]; + for (const control of controls) { + if (!control || control.querySelector(".simulated-label")) continue; + const marker = document.createElement("span"); + marker.className = "simulated-label"; + marker.textContent = "Simulated"; + marker.setAttribute("aria-hidden", "true"); + control.append(marker); + control.setAttribute("aria-label", `${control.textContent.trim()} (simulated)`); + } +} + +history.replaceState(null, "", `${location.pathname}${location.search}#token=static-demo`); +window.fetch = demoFetch; +markSimulatedControls(); +await import("../app.js"); diff --git a/packages/orchestration/operator/lib/runs.mjs b/packages/orchestration/operator/lib/runs.mjs index 4a1f508..2011c87 100644 --- a/packages/orchestration/operator/lib/runs.mjs +++ b/packages/orchestration/operator/lib/runs.mjs @@ -12,6 +12,7 @@ import { inspectRuntimeStateGuard, } from "../../scripts/pipeline/lib/runtime-state-guard.mjs"; import { validateRunId } from "./security.mjs"; +import { graphStatus, memoryStatus } from "../../scripts/pipeline/lib/graph.mjs"; const PHASES = [ "arm", @@ -105,11 +106,17 @@ function workspaceRuns(project, workspaceRoot, projectRoot) { function guardedWorkspaceRun(project, workspaceRoot) { const guard = inspectRuntimeStateGuard(workspaceRoot); - if (!guard.found) return null; - if (!guard.ownerActive) { - ensureRuntimeStateReadable(workspaceRoot, { expectedRunId: guard.runId }); - return null; - } + if (!guard.found || guardIsReadableInactive(guard, workspaceRoot)) return null; + return activeGuardedWorkspaceRun(project, workspaceRoot, guard); +} + +function guardIsReadableInactive(guard, workspaceRoot) { + if (guard.ownerActive) return false; + ensureRuntimeStateReadable(workspaceRoot, { expectedRunId: guard.runId }); + return true; +} + +function activeGuardedWorkspaceRun(project, workspaceRoot, guard) { return { id: guard.runId, project_id: project.id, @@ -180,20 +187,31 @@ function projectedEvents(run, runDir) { } function runTiming(request, events, runDir) { - const startedAt = - request.requested_at ?? - events[0]?.ts ?? - readJson(join(runDir, "operator-control.json"))?.updated_at ?? - null; + const startedAt = runStartTime(request, events, runDir); return { startedAt, - updatedAt: - readOperatorControl(basename(runDir), resolve(runDir, "../../..")).updated_at ?? - events.at(-1)?.ts ?? - startedAt, + updatedAt: runUpdatedTime(events, runDir, startedAt), }; } +function runStartTime(request, events, runDir) { + if (request.requested_at) return request.requested_at; + return eventOrControlStartTime(events, runDir); +} + +function eventOrControlStartTime(events, runDir) { + if (events[0]?.ts) return events[0].ts; + return readJson(join(runDir, "operator-control.json"))?.updated_at ?? null; +} + +function runUpdatedTime(events, runDir, startedAt) { + return ( + readOperatorControl(basename(runDir), resolve(runDir, "../../..")).updated_at ?? + events.at(-1)?.ts ?? + startedAt + ); +} + function runResources(progress, events) { return { input: progress.cost_summary?.total_tokens_in ?? null, @@ -212,6 +230,7 @@ function summarizeRun(project, run) { const { startedAt, updatedAt } = runTiming(request, events, runDir); const projectedArtifacts = events.filter((event) => event.event === "artifact_write"); const checkpointRows = checkpoints(runDir); + const workflow = workflowProjection(request, runDir); return { id: run.id, project_id: project.id, @@ -225,10 +244,109 @@ function summarizeRun(project, run) { evidence: { present: projectedArtifacts.length }, resources: runResources(progress, events), checkpoints: checkpointRows, + workflow, + graph_health: publicGraphHealth(run.workspaceRoot, run.id), workspaceRoot: run.workspaceRoot, }; } +function workflowProjection(request, runDir) { + if (request.workflow?.mode !== "graph-native") return null; + return { + workflow_id: request.workflow.workflow_id, + schema_version: request.workflow.snapshot?.schema_version ?? request.schema_version, + digest: request.workflow.digest, + revision: request.workflow.revision, + budgets: request.workflow.snapshot?.budgets ?? {}, + instances: workflowInstances(runDir), + }; +} + +function workflowInstances(runDir) { + const root = join(runDir, "workflow", "attempts"); + if (!existsSync(root)) return []; + const latest = new Map(); + for (const nodeEntry of readdirSync(root, { withFileTypes: true })) { + if (!nodeEntry.isDirectory()) continue; + collectLatestInstances(latest, join(root, nodeEntry.name)); + } + return [...latest.values()].sort(compareInstances); +} + +function collectLatestInstances(latest, directory) { + for (const name of readdirSync(directory).filter((entry) => entry.endsWith(".json"))) { + addLatestInstance(latest, readJson(join(directory, name))); + } +} + +function addLatestInstance(latest, envelope) { + if (!envelope || envelope.workflow_digest === undefined) return; + const instanceId = envelope.instance_id ?? envelope.node_id; + const prior = latest.get(instanceId); + if (!prior || envelope.attempt >= prior.attempt) + latest.set(instanceId, publicInstance(envelope, instanceId)); +} + +function publicInstance(envelope, instanceId) { + return { + instance_id: instanceId, + node_id: envelope.node_id, + parent_node: nullableProperty(envelope, "parent_node"), + item_key: nullableProperty(envelope, "item_key"), + item_digest: nullableProperty(envelope, "item_digest"), + status: envelope.status, + attempt: envelope.attempt, + execution_tier: propertyOr(envelope, "execution_tier", "runtime"), + selection: nullableProperty(envelope, "selection"), + quorum: nullableProperty(envelope, "quorum"), + convergence: nullableProperty(envelope, "convergence"), + }; +} + +function nullableProperty(record, key) { + return propertyOr(record, key, null); +} + +function propertyOr(record, key, fallback) { + return record[key] ?? fallback; +} + +function compareInstances(left, right) { + return left.instance_id.localeCompare(right.instance_id); +} + +function publicGraphHealth(workspaceRoot, runId) { + const status = graphStatus({ projectRoot: workspaceRoot, runId }); + const memory = graphMemoryHealth(workspaceRoot); + return { + ...publicGraphStatus(status), + stale_memory: memory.stale_facts ?? 0, + unresolved_conflicts: graphConflicts(status, memory), + }; +} + +function graphMemoryHealth(workspaceRoot) { + try { + return memoryStatus(workspaceRoot); + } catch { + return { stale_facts: 0, unresolved_conflicts: 1 }; + } +} + +function publicGraphStatus(status) { + return { + available: status.available, + valid: status.valid, + node_count: status.node_count ?? 0, + edge_count: status.edge_count ?? 0, + stale_sources: status.stale_sources ?? 0, + }; +} + +function graphConflicts(status, memory) { + return (status.unresolved_conflicts ?? 0) + (memory.unresolved_conflicts ?? 0); +} + function runIdentity(request, control, events, run) { return { task: runTask(request, run.id), diff --git a/packages/orchestration/operator/lib/workflows.mjs b/packages/orchestration/operator/lib/workflows.mjs new file mode 100644 index 0000000..cdeea66 --- /dev/null +++ b/packages/orchestration/operator/lib/workflows.mjs @@ -0,0 +1,48 @@ +/** Bridges the operator HTTP surface to the pipeline-owned workflow registry. */ +import { existsSync } from "node:fs"; +import { pathToFileURL } from "node:url"; +import { resolve } from "node:path"; + +const registryPath = resolve( + import.meta.dirname, + "../../scripts/pipeline/lib/workflow-registry.mjs", +); + +function unavailable() { + throw Object.assign(new Error("workflow registry is unavailable"), { status: 503 }); +} + +/** + * Loads the pipeline registry lazily so the console remains usable while an + * older checkout is upgraded. The registry module must export + * `createWorkflowRegistry(projectRoot)`, returning `{ list, show, draft, + * validate, diff, activate }`. Methods receive the workflow id first and + * accept a plain JSON request object where applicable. + */ +export async function workflowRegistryFor(project) { + if (!existsSync(registryPath)) + return Object.freeze({ + list: unavailable, + show: unavailable, + draft: unavailable, + validate: unavailable, + diff: unavailable, + activate: unavailable, + }); + const module = await import(pathToFileURL(registryPath).href); + if (typeof module.createWorkflowRegistry !== "function") + return Object.freeze({ + list: unavailable, + show: unavailable, + draft: unavailable, + validate: unavailable, + diff: unavailable, + activate: unavailable, + }); + return module.createWorkflowRegistry(project.root); +} + +export function assertRegistryMethod(registry, name) { + if (!registry || typeof registry[name] !== "function") unavailable(); + return registry[name].bind(registry); +} diff --git a/packages/orchestration/operator/server.mjs b/packages/orchestration/operator/server.mjs index 6744f66..2af9bb1 100644 --- a/packages/orchestration/operator/server.mjs +++ b/packages/orchestration/operator/server.mjs @@ -1,7 +1,6 @@ #!/usr/bin/env node /** Serves the authenticated loopback-only operator API and static console. */ import { createServer } from "node:http"; -import { readFileSync } from "node:fs"; import { dirname, extname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { RunController } from "./lib/control.mjs"; @@ -16,6 +15,7 @@ import { validateRunId, } from "./lib/security.mjs"; import { discoverRuns, locateRun, paginatedEvents, publicRun } from "./lib/runs.mjs"; +import { assertRegistryMethod, workflowRegistryFor } from "./lib/workflows.mjs"; import { assertSupportedNodeRuntime } from "../scripts/lib/node-runtime.mjs"; assertSupportedNodeRuntime(); @@ -32,6 +32,7 @@ const STATIC_ROOT_FILES = new Map([ ["/index.html", "index.html"], ]); const API_PREFIX = "/api/v1"; +const { readFileSync } = process.getBuiltinModule("node:fs"); function securityHeaders() { return { @@ -162,52 +163,88 @@ async function routeApi(req, res, url, context) { sendJson(res, 401, { error: { status: 401, message: "bearer authentication required" } }); return; } + return routeAuthorizedApi(req, res, url, context, projects, controller); +} +async function routeAuthorizedApi(req, res, url, context, projects, controller) { const parts = splitPath(url.pathname); - if (parts[0] !== "api" || parts[1] !== "v1") - throw Object.assign(new Error("not found"), { status: 404 }); - if (parts.length === 3 && parts[2] === "projects") { - requireMethod(req, "GET"); - sendJson(res, 200, { - projects: projects.map(({ id, label }) => ({ id, label })), - active_run_id: controller.refreshOwnership(), - }); - return; + assertApiPath(parts); + if (isProjectsEndpoint(parts)) { + return routeProjects(req, res, projects, controller); + } + const project = projectForRoute(projects, parts); + if (parts[4] === "workflows") { + return routeWorkflows(req, res, url, context, project, parts.slice(5)); } + if (parts[4] !== "runs") throw Object.assign(new Error("not found"), { status: 404 }); + return routeRuns(req, res, url, project, controller, parts); +} + +function assertApiPath(parts) { + if (parts[0] === "api" && parts[1] === "v1") return; + throw Object.assign(new Error("not found"), { status: 404 }); +} + +function isProjectsEndpoint(parts) { + return parts.length === 3 && parts[2] === "projects"; +} + +function projectForRoute(projects, parts) { if (parts[2] !== "projects" || !parts[3]) throw Object.assign(new Error("not found"), { status: 404 }); const project = findProject(projects, parts[3]); if (!project) throw Object.assign(new Error("project not found"), { status: 404 }); - if (parts[4] !== "runs") throw Object.assign(new Error("not found"), { status: 404 }); + return project; +} +function routeProjects(req, res, projects, controller) { + requireMethod(req, "GET"); + sendJson(res, 200, { + projects: projects.map(({ id, label }) => ({ id, label })), + active_run_id: controller.refreshOwnership(), + }); +} + +async function routeRuns(req, res, url, project, controller, parts) { if (parts.length === 5) { - if (req.method === "GET") { - const cursor = positiveInteger(url.searchParams.get("cursor"), 0, 1_000_000); - const limit = positiveInteger(url.searchParams.get("limit"), 30, 100); - controller.refreshOwnership(); - const all = discoverRuns(project); - const page = all - .slice(cursor, cursor + limit) - .map((run) => publicRun(run, controller.ownedRunId)); - sendJson(res, 200, { - runs: page, - next_cursor: cursor + page.length < all.length ? cursor + page.length : null, - }); - return; - } - requireMethod(req, "POST"); - sendJson(res, 202, controller.start(project, await readJsonBody(req))); - return; + return routeRunCollection(req, res, url, project, controller); } - const runId = validateRunId(parts[5]); if (parts.length === 6) { - requireMethod(req, "GET"); - controller.refreshOwnership(); - sendJson(res, 200, { run: publicRun(locateRun(project, runId), controller.ownedRunId) }); - return; + return routeRunDetail(req, res, project, controller, runId); } + return routeRunAction(req, res, url, project, controller, runId, parts); +} + +function runPage(url, project, controller) { + const cursor = positiveInteger(url.searchParams.get("cursor"), 0, 1_000_000); + const limit = positiveInteger(url.searchParams.get("limit"), 30, 100); + controller.refreshOwnership(); + const all = discoverRuns(project); + const runs = all + .slice(cursor, cursor + limit) + .map((run) => publicRun(run, controller.ownedRunId)); + const nextCursor = cursor + runs.length < all.length ? cursor + runs.length : null; + return { runs, next_cursor: nextCursor }; +} + +async function routeRunCollection(req, res, url, project, controller) { + if (req.method === "GET") return sendJson(res, 200, runPage(url, project, controller)); + return startRun(req, res, project, controller); +} + +async function startRun(req, res, project, controller) { + requireMethod(req, "POST"); + sendJson(res, 202, controller.start(project, await readJsonBody(req))); +} + +function routeRunDetail(req, res, project, controller, runId) { + requireMethod(req, "GET"); + controller.refreshOwnership(); + sendJson(res, 200, { run: publicRun(locateRun(project, runId), controller.ownedRunId) }); +} +async function routeRunAction(req, res, url, project, controller, runId, parts) { const action = parts[6]; if (action === "events" && parts.length === 7) { requireMethod(req, "GET"); @@ -225,19 +262,105 @@ async function routeApi(req, res, url, context) { requireMethod(req, "POST"); if (parts.length !== 7) throw Object.assign(new Error("not found"), { status: 404 }); const body = await readJsonBody(req); - if (action === "stop") { - sendJson(res, 200, { control: controller.stop(project, runId) }); - } else if (action === "resume") { - sendJson(res, 202, controller.resume(project, runId)); - } else if (action === "interrupt") { - sendJson(res, 202, controller.interrupt(project, runId, body)); - } else if (action === "checkpoint-decision") { - sendJson(res, 200, { checkpoint: controller.decideCheckpoint(project, runId, body) }); - } else if (action === "cleanup") { - sendJson(res, 202, controller.cleanup(project, runId, body)); - } else { - throw Object.assign(new Error("not found"), { status: 404 }); + return routeRunControlAction(res, action, project, controller, runId, body); +} + +function routeRunControlAction(res, action, project, controller, runId, body) { + switch (action) { + case "stop": + return sendJson(res, 200, { control: controller.stop(project, runId) }); + case "resume": + return sendJson(res, 202, controller.resume(project, runId)); + case "interrupt": + return sendJson(res, 202, controller.interrupt(project, runId, body)); + case "checkpoint-decision": + return sendJson(res, 200, { checkpoint: controller.decideCheckpoint(project, runId, body) }); + case "cleanup": + return sendJson(res, 202, controller.cleanup(project, runId, body)); + default: + throw Object.assign(new Error("not found"), { status: 404 }); + } +} + +function projectHasActiveRun(project) { + try { + return discoverRuns(project).some( + (run) => run.runtime_active || ["running", "waiting", "stop-requested"].includes(run.status), + ); + } catch { + // A missing or unreadable runtime directory is not an active execution. + return false; + } +} + +function workflowMutationAllowed(project, controller) { + controller.refreshOwnership(); + if (controller.ownedRunId || projectHasActiveRun(project)) { + throw Object.assign(new Error("workflow revisions are immutable while a run is active"), { + status: 409, + }); + } +} + +async function routeWorkflows(req, res, url, context, project, tail) { + const registry = context.workflowRegistry ?? (await workflowRegistryFor(project)); + if (tail.length === 0) { + requireMethod(req, "GET"); + sendJson(res, 200, { workflows: await assertRegistryMethod(registry, "list")() }); + return; + } + const workflowId = tail[0]; + if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(workflowId)) { + throw Object.assign(new Error("invalid workflow id"), { status: 400 }); + } + if (tail.length === 1) { + requireMethod(req, "GET"); + sendJson(res, 200, { workflow: await assertRegistryMethod(registry, "show")(workflowId) }); + return; + } + if (tail[1] === "drafts" && tail.length === 2) { + requireMethod(req, "POST"); + workflowMutationAllowed(project, context.controller); + sendJson(res, 201, { + revision: await assertRegistryMethod(registry, "draft")(workflowId, await readJsonBody(req)), + }); + return; + } + if (tail[1] === "diff" && tail.length === 2) { + requireMethod(req, "GET"); + sendJson(res, 200, { + diff: await assertRegistryMethod(registry, "diff")( + workflowId, + Object.fromEntries(url.searchParams), + ), + }); + return; + } + if (tail[1] === "revisions" && tail[3] === "validate" && tail.length === 4) { + requireMethod(req, "POST"); + workflowMutationAllowed(project, context.controller); + sendJson(res, 200, { + validation: await assertRegistryMethod(registry, "validate")( + workflowId, + tail[2], + await readJsonBody(req), + ), + }); + return; + } + if (tail[1] === "revisions" && tail[3] === "activate" && tail.length === 4) { + requireMethod(req, "POST"); + workflowMutationAllowed(project, context.controller); + sendJson(res, 200, { + activation: await assertRegistryMethod(registry, "activate")( + workflowId, + tail[2], + await readJsonBody(req), + ), + }); + return; } + throw Object.assign(new Error("not found"), { status: 404 }); } export async function handleOperatorRequest(req, res, context) { @@ -286,29 +409,31 @@ export function createOperatorServer({ function parseCli(argv) { const paths = []; let port = 0; - for (let index = 0; index < argv.length; index += 1) { - const arg = argv[index]; - if (arg === "--project") { - const value = argv[index + 1]; + const args = argv.values(); + for (let arg = args.next(); !arg.done; arg = args.next()) { + if (arg.value === "--project") { + const value = args.next().value; if (!value) throw new Error("--project requires a path"); paths.push(value); - index += 1; - } else if (arg === "--port") { - port = Number(argv[index + 1]); + } else if (arg.value === "--port") { + port = Number(args.next().value); if (!Number.isInteger(port) || port < 0 || port > 65535) throw new Error("invalid --port"); - index += 1; - } else if (arg === "--help" || arg === "-h") { - process.stdout.write( - "Usage: node operator/server.mjs --project [--project ] [--port 0]\n", - ); + } else if (arg.value === "--help" || arg.value === "-h") { + writeUsage(); return null; } else { - throw new Error(`unknown argument: ${arg}`); + throw new Error(`unknown argument: ${arg.value}`); } } return { projects: createProjectRegistry(paths), port }; } +function writeUsage() { + process.stdout.write( + "Usage: node operator/server.mjs --project PROJECT_ROOT [--project PROJECT_ROOT] [--port PORT]\n", + ); +} + async function main() { const options = parseCli(process.argv.slice(2)); if (!options) return; diff --git a/packages/orchestration/operator/static/app.js b/packages/orchestration/operator/static/app.js index 837413f..2225d55 100644 --- a/packages/orchestration/operator/static/app.js +++ b/packages/orchestration/operator/static/app.js @@ -2,6 +2,7 @@ import { setConnection, showError } from "./js/api.js"; import { loadProjects } from "./js/data.js"; import { bindHandlers } from "./js/handlers.js"; +import { bindWorkflowEditor } from "./js/workflows.js"; import { elements, state } from "./js/state.js"; // Contract anchors retained for source-level UI tests and operator safety review. @@ -15,6 +16,7 @@ void [ ]; bindHandlers(); +bindWorkflowEditor(); loadProjects().catch((error) => { setConnection("error", "Unavailable", "Local session"); diff --git a/packages/orchestration/operator/static/css/workflows.css b/packages/orchestration/operator/static/css/workflows.css new file mode 100644 index 0000000..80b7a13 --- /dev/null +++ b/packages/orchestration/operator/static/css/workflows.css @@ -0,0 +1,30 @@ +/* Revision editor: semantic controls are usable without graph interpretation. */ +.workflow-editor { margin-top: 2rem; padding: 1.25rem; border: 1px solid var(--line, #c7d0d4); background: var(--panel, #fff); } +.workflow-editor__head { display:flex; justify-content:space-between; gap:1rem; align-items:center; } +.workflow-editor__head h2 { margin:.15rem 0 0; } +.workflow-editor__grid { display:grid; grid-template-columns:minmax(10rem, .7fr) minmax(18rem, 2fr) minmax(14rem, 1fr); gap:1rem; margin-top:1rem; } +.workflow-rail,.workflow-inspector { border:1px solid var(--line, #c7d0d4); padding:.8rem; min-width:0; } +.workflow-rail h3,.workflow-inspector h3 { margin-top:0; } +.workflow-canvas { min-width:0; } +#workflow-graph { display:block; width:100%; min-height:14rem; border:1px solid var(--line, #c7d0d4); background:#f7fafb; } +.workflow-node { fill:#fff; stroke:#17324d; stroke-width:2; rx:6; } +.workflow-node--join,.workflow-node--fan-out { fill:#e7f3f6; } +.workflow-node--gate,.workflow-node--checkpoint { fill:#fff4dc; } +.workflow-node--terminal { fill:#e8f4ea; } +.workflow-edge { stroke:#17324d; stroke-width:2; fill:none; marker-end:url(#workflow-arrow); } +.workflow-edge--artifact,.workflow-edge--stream { stroke:#006b75; } +.workflow-edge--stream { stroke-dasharray:3 3; stroke-width:3; } +.workflow-edge--condition { stroke:#995600; stroke-dasharray:7 4; } +.workflow-edge--loop-back { stroke:#7a3e98; stroke-dasharray:2 4; } +.workflow-edge-label { fill:#435466; font:10px ui-monospace,monospace; text-anchor:middle; paint-order:stroke; stroke:#f7fafb; stroke-width:3px; } +.workflow-node-badge { fill:#506274; font:10px ui-monospace,monospace; } +.workflow-form { display:grid; gap:.55rem; margin-top:.75rem; } +.workflow-form textarea,#workflow-diff-output { width:100%; box-sizing:border-box; font: .82rem ui-monospace, monospace; } +.workflow-form textarea { min-height:9rem; } +.workflow-rail button { display:block; width:100%; margin:.25rem 0; text-align:left; } +#workflow-history { padding-left:1.2rem; } +.workflow-structure { display:grid; gap:.65rem; max-height:22rem; overflow:auto; } +.workflow-structure table { width:100%; border-collapse:collapse; font-size:.82rem; } +.workflow-structure th,.workflow-structure td { padding:.35rem; border-bottom:1px solid var(--line, #c7d0d4); text-align:left; vertical-align:top; } +#workflow-diff-output { min-height:5rem; white-space:pre-wrap; border:1px solid var(--line, #c7d0d4); padding:.5rem; } +@media (max-width: 900px) { .workflow-editor__grid { grid-template-columns:1fr; } .workflow-rail { max-height:12rem; overflow:auto; } } diff --git a/packages/orchestration/operator/static/index.html b/packages/orchestration/operator/static/index.html index a9e3062..ce78d0e 100644 --- a/packages/orchestration/operator/static/index.html +++ b/packages/orchestration/operator/static/index.html @@ -284,6 +284,54 @@

Evidence log

+ +
+
+

Revision-controlled

Workflow systems map

+ +
+
+ +
+ + Workflow stage map + A topologically layered workflow map. Color and line style distinguish data, stream, condition, sequence, and loop-back edges. The tables after the map provide the complete equivalent structure and live instance state. + + + +
+ + + + + + +

Revision budget unavailable

+
+
+
+

Structured nodes and edges

+
+
+
+ +
+

Drafts cannot change while a run is active.

+
diff --git a/packages/orchestration/operator/static/js/data.js b/packages/orchestration/operator/static/js/data.js index 2da8ef1..59f4c38 100644 --- a/packages/orchestration/operator/static/js/data.js +++ b/packages/orchestration/operator/static/js/data.js @@ -3,6 +3,7 @@ import { api, setConnection, showError } from "./api.js"; import { elements, state } from "./state.js"; import { renderEvents, renderRun, renderRuns } from "./render.js"; +import { loadWorkflows } from "./workflows.js"; export async function loadProjects() { const data = await api("/projects"); @@ -15,6 +16,7 @@ export async function loadProjects() { setConnection("connected", "Local session", "No publish controls"); if (state.projectId) { await loadRuns(); + await loadWorkflows(); } else { elements["runs-loading"].hidden = true; renderRuns(); @@ -106,8 +108,7 @@ async function streamEvents(after, generation, projectId, runId) { if (!isCurrentEventSelection(generation, projectId, runId)) return; const controller = new AbortController(); state.streamAbort = controller; - elements["stream-status"].innerHTML = - ` Live verification`; + showLiveVerificationStatus(); const response = await openEventStream(after, controller, projectId, runId); if (!isCurrentEventSelection(generation, projectId, runId)) { controller.abort(); @@ -121,7 +122,7 @@ async function streamEvents(after, generation, projectId, runId) { } async function openEventStream(after, controller, projectId, runId) { - return await fetch( + return fetch( `/api/v1/projects/${encodeURIComponent(projectId)}/runs/${encodeURIComponent(runId)}/events/stream?after=${after}`, { headers: { authorization: `Bearer ${state.token}` }, @@ -130,18 +131,30 @@ async function openEventStream(after, controller, projectId, runId) { ); } +function showLiveVerificationStatus() { + const status = elements["stream-status"]; + const spinner = document.createElement("span"); + spinner.className = "spinner"; + spinner.setAttribute("aria-hidden", "true"); + status.replaceChildren(spinner, document.createTextNode(" Live verification")); +} + async function consumeEventStream(response, generation, projectId, runId) { const reader = response.body.getReader(); const decoder = new TextDecoder(); let buffer = ""; - for (;;) { - const { value, done } = await reader.read(); - if (done || !isCurrentEventSelection(generation, projectId, runId)) break; - buffer += decoder.decode(value, { stream: true }); - const lines = buffer.split("\n"); - buffer = lines.pop(); - appendStreamEvents(lines); - renderEvents(); + try { + for (;;) { + const { value, done } = await reader.read(); + if (done || !isCurrentEventSelection(generation, projectId, runId)) break; + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split("\n"); + buffer = lines.pop(); + appendStreamEvents(lines); + renderEvents(); + } + } catch (error) { + if (error.name !== "AbortError") throw error; } } diff --git a/packages/orchestration/operator/static/js/format.js b/packages/orchestration/operator/static/js/format.js index f2afcf9..5d44ee3 100644 --- a/packages/orchestration/operator/static/js/format.js +++ b/packages/orchestration/operator/static/js/format.js @@ -1,20 +1,19 @@ /** Pure formatting helpers for the operator console. */ export function phaseLabel(phase) { - return ( - { - arm: "Intake", - design: "Design", - "adversarial-review": "Adversarial review", - plan: "Plan", - pmatch: "Drift match", - build: "Build", - "quality-static": "Quality static", - "quality-tests": "Quality tests", - "post-build": "Post-build", - "release-readiness": "Release readiness", - }[phase] ?? humanize(phase) - ); + const labels = new Map([ + ["arm", "Intake"], + ["design", "Design"], + ["adversarial-review", "Adversarial review"], + ["plan", "Plan"], + ["pmatch", "Drift match"], + ["build", "Build"], + ["quality-static", "Quality static"], + ["quality-tests", "Quality tests"], + ["post-build", "Post-build"], + ["release-readiness", "Release readiness"], + ]); + return labels.get(phase) ?? humanize(phase); } export function humanize(value) { @@ -106,8 +105,10 @@ export function relativeTime(value) { const delta = Date.now() - date.valueOf(); if (delta < 60_000) return "Now"; if (delta < 3_600_000) return `${Math.floor(delta / 60_000)}m`; - if (delta < 86_400_000) return `${Math.floor(delta / 3_600_000)}h`; - if (delta < 172_800_000) return "Yesterday"; + const hourMs = 60 * 60 * 1000; + const dayMs = 24 * hourMs; + if (delta < dayMs) return `${Math.floor(delta / hourMs)}h`; + if (delta < 2 * dayMs) return "Yesterday"; return date.toLocaleDateString([], { month: "short", day: "numeric" }); } diff --git a/packages/orchestration/operator/static/js/handlers.js b/packages/orchestration/operator/static/js/handlers.js index 33d1c20..cfb18a9 100644 --- a/packages/orchestration/operator/static/js/handlers.js +++ b/packages/orchestration/operator/static/js/handlers.js @@ -5,6 +5,7 @@ import { closeConfirmation, openConfirmation, postAction, submitConfirmation } f import { loadRuns, selectRun, waitForNewRun } from "./data.js"; import { humanize } from "./format.js"; import { renderRuns, setEvidenceExpanded } from "./render.js"; +import { loadWorkflows } from "./workflows.js"; import { currentRun, elements, state } from "./state.js"; export function bindHandlers() { @@ -14,6 +15,7 @@ export function bindHandlers() { state.runQuery = ""; elements["run-search-input"].value = ""; await loadRuns(false).catch(showError); + await loadWorkflows().catch(showError); }); elements["toggle-search"].addEventListener("click", () => { @@ -63,8 +65,8 @@ export function bindHandlers() { elements["start-task"].focus(); }); - for (const id of ["start-close", "start-cancel"]) { - elements[id].addEventListener("click", () => elements["start-dialog"].close()); + for (const element of [elements["start-close"], elements["start-cancel"]]) { + element.addEventListener("click", () => elements["start-dialog"].close()); } elements["start-form"].addEventListener("submit", async (event) => { @@ -116,8 +118,8 @@ export function bindHandlers() { elements["confirm-run-id"].value !== elements["confirm-expected-id"].textContent; }); - for (const id of ["confirm-close", "confirm-cancel"]) { - elements[id].addEventListener("click", closeConfirmation); + for (const element of [elements["confirm-close"], elements["confirm-cancel"]]) { + element.addEventListener("click", closeConfirmation); } elements["confirm-form"].addEventListener("submit", async (event) => { diff --git a/packages/orchestration/operator/static/js/render.js b/packages/orchestration/operator/static/js/render.js index 95bd57d..7f0bb8a 100644 --- a/packages/orchestration/operator/static/js/render.js +++ b/packages/orchestration/operator/static/js/render.js @@ -92,13 +92,13 @@ export function renderRuns() { } export function renderRunControls(run) { - for (const [element, key] of [ - [elements["stop-button"], "stop"], - [elements["interrupt-button"], "interrupt"], - [elements["resume-button"], "resume"], - [elements["cleanup-button"], "cleanup"], + for (const [element, enabled] of [ + [elements["stop-button"], run?.controls?.stop], + [elements["interrupt-button"], run?.controls?.interrupt], + [elements["resume-button"], run?.controls?.resume], + [elements["cleanup-button"], run?.controls?.cleanup], ]) { - element.disabled = !run?.controls[key]; + element.disabled = !enabled; } } diff --git a/packages/orchestration/operator/static/js/state.js b/packages/orchestration/operator/static/js/state.js index 8e8e3ee..25f3aa7 100644 --- a/packages/orchestration/operator/static/js/state.js +++ b/packages/orchestration/operator/static/js/state.js @@ -14,6 +14,9 @@ export const state = { runsGeneration: 0, streamAbort: null, streamGeneration: 0, + workflows: [], + workflowId: null, + workflow: null, }; export const elements = Object.fromEntries( @@ -87,6 +90,27 @@ export const elements = Object.fromEntries( "confirm-submit", "confirm-close", "confirm-cancel", + "workflow-refresh", + "workflow-list", + "workflow-empty", + "workflow-graph", + "workflow-graph-content", + "workflow-definition", + "workflow-actor", + "workflow-rationale", + "workflow-budget", + "workflow-structure", + "workflow-draft-form", + "workflow-draft", + "workflow-validate", + "workflow-details", + "workflow-diff-base", + "workflow-diff", + "workflow-diff-output", + "workflow-activate", + "workflow-digest-confirmation", + "workflow-history", + "workflow-status", ].map((id) => [id, document.getElementById(id)]), ); diff --git a/packages/orchestration/operator/static/js/workflows.js b/packages/orchestration/operator/static/js/workflows.js new file mode 100644 index 0000000..62ccff0 --- /dev/null +++ b/packages/orchestration/operator/static/js/workflows.js @@ -0,0 +1,500 @@ +/** Renders and controls the authenticated workflow registry editor. */ +import { api, showError, showToast } from "./api.js"; +import { currentRun, elements, state } from "./state.js"; + +const escapeText = (value) => String(value ?? ""); +const base = () => `/projects/${encodeURIComponent(state.projectId)}/workflows`; +const firstDefined = (...values) => values.find((value) => value !== undefined); + +function selectedRevision() { + return state.workflow?.revisions?.at(-1)?.revision ?? state.workflow?.workflow?.revision ?? null; +} + +function neighborIds(edges, nodeId, direction) { + return edges + .filter((edge) => (direction === "to" ? edge.to : edge.from) === nodeId) + .map((edge) => (direction === "to" ? edge.from : edge.to)) + .sort(); +} + +function isCollapsibleNode(node) { + return node.kind === "agent" && node.access === "read"; +} + +function fanoutSignatures(definition) { + const edges = definition.edges ?? []; + const signatures = new Map(); + for (const node of definition.nodes ?? []) { + if (!isCollapsibleNode(node)) continue; + const incoming = neighborIds(edges, node.id, "to"); + const outgoing = neighborIds(edges, node.id, "from"); + const signature = `${incoming.join(",")}|${outgoing.join(",")}`; + if (!signatures.has(signature)) signatures.set(signature, []); + signatures.get(signature).push(node); + } + return new Map( + [...signatures.values()] + .filter((group) => group.length > 2) + .flatMap((group) => group.map((node) => [node.id, group])), + ); +} + +function fanoutVertex(group) { + return { + id: `fanout:${group + .map((item) => item.id) + .sort() + .join("+")}`, + kind: "fan-out", + access: "read", + tier: group.every((item) => item.tier === group[0].tier) ? group[0].tier : "mixed", + members: group.map((item) => item.id).sort(), + }; +} + +function groupedVertices(definition) { + const collapsed = fanoutSignatures(definition); + const emitted = new Set(); + const vertices = []; + const aliases = new Map(); + for (const node of definition.nodes ?? []) { + const group = collapsed.get(node.id); + if (!group) { + vertices.push({ ...node, members: [node.id] }); + aliases.set(node.id, node.id); + continue; + } + const vertex = fanoutVertex(group); + aliases.set(node.id, vertex.id); + if (emitted.has(vertex.id)) continue; + emitted.add(vertex.id); + vertices.push(vertex); + } + return { vertices, aliases }; +} + +function topology(definition) { + const { vertices, aliases } = groupedVertices(definition); + const edges = normalizedEdges(definition.edges ?? [], vertices, aliases); + return { vertices, edges, layers: topologyLayers(vertices, edges) }; +} + +function normalizedEdges(sourceEdges, vertices, aliases) { + const ids = new Set(vertices.map(({ id }) => id)); + const seen = new Set(); + return sourceEdges.flatMap((edge) => uniqueEdge(edge, ids, aliases, seen)); +} + +function uniqueEdge(edge, ids, aliases, seen) { + const from = firstDefined(aliases.get(edge.from), edge.from); + const to = firstDefined(aliases.get(edge.to), edge.to); + if (!ids.has(from) || !ids.has(to) || from === to) return []; + const detail = firstDefined(edge.condition, edge.artifact, ""); + const key = `${from}|${to}|${edge.type}|${detail}`; + if (seen.has(key)) return []; + seen.add(key); + return [{ ...edge, from, to }]; +} + +function topologyLayers(vertices, edges) { + const depth = new Map(vertices.map(({ id }) => [id, 0])); + const acyclic = edges.filter((edge) => edge.type !== "loop-back"); + const incoming = new Map(vertices.map(({ id }) => [id, 0])); + for (const edge of acyclic) incoming.set(edge.to, (incoming.get(edge.to) ?? 0) + 1); + const queue = [...vertices.filter(({ id }) => incoming.get(id) === 0).map(({ id }) => id)].sort(); + while (queue.length) { + const id = queue.shift(); + advanceTopologyDepth(id, acyclic, depth, incoming, queue); + queue.sort(); + } + return groupedLayers(vertices, depth); +} + +function advanceTopologyDepth(id, edges, depth, incoming, queue) { + for (const edge of edges.filter((candidate) => candidate.from === id)) { + depth.set(edge.to, Math.max(depth.get(edge.to), depth.get(id) + 1)); + incoming.set(edge.to, incoming.get(edge.to) - 1); + if (incoming.get(edge.to) === 0) queue.push(edge.to); + } +} + +function groupedLayers(vertices, depth) { + const layers = new Map(); + for (const vertex of vertices) { + const layer = depth.get(vertex.id) ?? 0; + if (!layers.has(layer)) layers.set(layer, []); + layers.get(layer).push(vertex); + } + for (const layer of layers.values()) layer.sort((left, right) => left.id.localeCompare(right.id)); + return layers; +} + +function renderGraph(definition = {}) { + const graph = elements["workflow-graph-content"]; + graph.replaceChildren(); + const { vertices, edges, layers } = topology(definition); + setGraphViewBox(layers); + const positions = graphPositions(layers); + renderEdges(graph, edges, positions); + renderNodes(graph, vertices, positions); +} + +function setGraphViewBox(layers) { + const maximumRows = Math.max(1, ...[...layers.values()].map((layer) => layer.length)); + const maximumDepth = Math.max(0, ...layers.keys()); + elements["workflow-graph"].setAttribute( + "viewBox", + `0 0 ${Math.max(640, (maximumDepth + 1) * 180 + 30)} ${Math.max(250, maximumRows * 92 + 45)}`, + ); +} + +function graphPositions(layers) { + const positions = new Map(); + for (const [layerIndex, layer] of layers) { + layer.forEach((node, row) => { + positions.set(node.id, { x: 20 + layerIndex * 180, y: 24 + row * 92 }); + }); + } + return positions; +} + +function renderEdges(graph, edges, positions) { + for (const edgeRecord of edges) { + const from = positions.get(edgeRecord.from); + const to = positions.get(edgeRecord.to); + if (!from || !to) continue; + graph.append(edgePath(edgeRecord, from, to), edgeLabel(edgeRecord, from, to)); + } +} + +function edgePath(edgeRecord, from, to) { + const edge = document.createElementNS("http://www.w3.org/2000/svg", "path"); + edge.setAttribute("class", `workflow-edge workflow-edge--${edgeRecord.type}`); + edge.setAttribute("d", edgePathData(edgeRecord.type, from, to)); + return edge; +} + +function edgePathData(type, from, to) { + const bend = type === "loop-back" ? Math.min(from.y, to.y) - 18 : (from.x + to.x) / 2; + if (type === "loop-back") + return `M${from.x + 130} ${from.y + 30} C${from.x + 155} ${bend} ${to.x - 25} ${bend} ${to.x} ${to.y + 30}`; + return `M${from.x + 130} ${from.y + 30} C${bend} ${from.y + 30} ${bend} ${to.y + 30} ${to.x} ${to.y + 30}`; +} + +function edgeLabel(edgeRecord, from, to) { + const label = document.createElementNS("http://www.w3.org/2000/svg", "text"); + label.setAttribute("class", "workflow-edge-label"); + label.setAttribute("x", (from.x + to.x + 130) / 2); + label.setAttribute("y", (from.y + to.y) / 2 + 20); + label.textContent = edgeRecord.condition ?? edgeRecord.artifact ?? edgeRecord.type; + return label; +} + +function renderNodes(graph, vertices, positions) { + for (const node of vertices) { + const position = positions.get(node.id); + if (position) + graph.append( + nodeRectangle(node, position), + nodeText(node, position), + nodeBadge(node, position), + ); + } +} + +function nodeRectangle(node, { x, y }) { + const rect = document.createElementNS("http://www.w3.org/2000/svg", "rect"); + rect.setAttribute("class", `workflow-node workflow-node--${node.kind}`); + rect.setAttribute("x", x); + rect.setAttribute("y", y); + rect.setAttribute("width", "130"); + rect.setAttribute("height", "60"); + return rect; +} + +function nodeText(node, { x, y }) { + const text = document.createElementNS("http://www.w3.org/2000/svg", "text"); + text.setAttribute("x", x + 8); + text.setAttribute("y", y + 24); + text.textContent = escapeText( + node.kind === "fan-out" ? `${node.members.length} parallel nodes` : node.id, + ); + return text; +} + +function nodeBadge(node, { x, y }) { + const badge = document.createElementNS("http://www.w3.org/2000/svg", "text"); + badge.setAttribute("class", "workflow-node-badge"); + badge.setAttribute("x", x + 8); + badge.setAttribute("y", y + 46); + badge.textContent = `${node.kind} · ${node.tier ?? node.access}`; + return badge; +} + +function structuredTable(caption, columns, rows) { + const table = document.createElement("table"); + const captionElement = document.createElement("caption"); + captionElement.textContent = caption; + const head = document.createElement("thead"); + const headingRow = document.createElement("tr"); + for (const column of columns) { + const cell = document.createElement("th"); + cell.scope = "col"; + cell.textContent = column; + headingRow.append(cell); + } + head.append(headingRow); + const body = document.createElement("tbody"); + for (const row of rows) { + const tableRow = document.createElement("tr"); + for (const value of row) { + const cell = document.createElement("td"); + cell.textContent = escapeText(value); + tableRow.append(cell); + } + body.append(tableRow); + } + table.append(captionElement, head, body); + return table; +} + +function renderStructure(definition) { + const instances = visibleInstances(definition); + elements["workflow-structure"].replaceChildren( + structuredTable( + "Nodes", + ["ID", "Kind", "Access", "Tier", "Join or fan-out", "Failure"], + (definition.nodes ?? []).map(nodeStructureRow), + ), + structuredTable( + "Edges", + ["From", "To", "Type", "Artifact or condition"], + (definition.edges ?? []).map(edgeStructureRow), + ), + structuredTable( + "Live and completed node instances", + ["Instance", "Node", "Status", "Attempt", "Item key", "Tier", "Decision"], + instances.map(instanceStructureRow), + ), + ); +} + +function visibleInstances(definition) { + const run = currentRun(); + const matches = + run?.workflow?.workflow_id === definition.workflow_id || + run?.workflow?.digest === state.workflow?.digest; + return matches ? (run.workflow.instances ?? []) : []; +} + +function nodeStructureRow(node) { + return [ + node.id, + node.kind, + node.access, + node.tier ?? "standard", + nodeJoinLabel(node), + node.failure_handling?.mode ?? "fail-workflow", + ]; +} + +function nodeJoinLabel(node) { + if (node.join === "quorum") return `quorum ${node.quorum?.threshold ?? "?"}`; + return node.join ?? (node.map ? `map ≤${node.map.max_items ?? 32}` : "—"); +} + +function edgeStructureRow(edge) { + return [edge.from, edge.to, edge.type, edge.condition ?? edge.artifact ?? "—"]; +} + +function instanceStructureRow(instance) { + return [ + instance.instance_id, + instance.node_id, + instance.status, + instance.attempt, + instance.item_key ?? "—", + instance.execution_tier, + instanceDecisionLabel(instance), + ]; +} + +function instanceDecisionLabel(instance) { + if (instance.selection) return `selected ${instance.selection.winner ?? "input"}`; + if (instance.quorum) + return `quorum ${instance.quorum.passed ?? instance.quorum.accepted ?? "—"}/${instance.quorum.threshold ?? "—"}`; + return instance.convergence?.dry ? "converged" : "—"; +} + +function mutationLocked() { + return state.runs.some( + (run) => run.runtime_active || ["running", "waiting", "phase-active"].includes(run.status), + ); +} + +function updateMutationControls() { + const locked = mutationLocked(); + for (const control of mutationControls()) { + control.disabled = locked; + } + elements["workflow-definition"].readOnly = locked; + if (locked) + elements["workflow-status"].textContent = "Registry is read-only while a run is active."; +} + +function mutationControls() { + return [elements["workflow-draft"], elements["workflow-validate"], elements["workflow-activate"]]; +} + +function renderWorkflowBudget(definition) { + elements["workflow-budget"].textContent = definition.budgets + ? `Budgets: concurrency ${definition.budgets.max_concurrency ?? 4}; map ${definition.budgets.max_map_items ?? 32}; pipeline depth ${definition.budgets.max_pipeline_depth ?? 4}; dynamic instances or attempts ${definition.budgets.max_dynamic_instances ?? 128}; repair rounds ${definition.budgets.max_repair_rounds ?? 5}.` + : "No explicit revision budgets."; +} + +function renderWorkflowDetails(workflow) { + elements["workflow-details"].replaceChildren( + ...Object.entries({ + id: workflow.workflow_id, + active_revision: + workflow.active?.workflow_id === workflow.workflow_id ? workflow.active.revision : "none", + latest_revision: selectedRevision() ?? "none", + digest: workflow.digest, + }).map(([key, value]) => { + const box = document.createElement("div"), + dt = document.createElement("dt"), + dd = document.createElement("dd"); + dt.textContent = key.replace("_", " "); + dd.textContent = escapeText(value); + box.append(dt, dd); + return box; + }), + ); +} + +function renderWorkflowHistory(workflow) { + const history = workflow.activation_history ?? []; + elements["workflow-history"].replaceChildren( + ...history.map((item) => { + const li = document.createElement("li"); + li.textContent = + typeof item === "string" + ? item + : `${item.revision ?? "revision"} · ${item.activated_at ?? "recorded"}`; + return li; + }), + ); +} + +function renderWorkflow() { + const workflow = state.workflow; + if (!workflow) return; + const definition = workflow.workflow ?? {}; + elements["workflow-definition"].value = JSON.stringify(definition, null, 2); + renderWorkflowBudget(definition); + renderWorkflowDetails(workflow); + renderWorkflowHistory(workflow); + renderGraph(definition); + renderStructure(definition); + updateMutationControls(); +} + +async function selectWorkflow(id) { + state.workflowId = id; + state.workflow = (await api(`${base()}/${encodeURIComponent(id)}`)).workflow; + renderWorkflow(); + [...elements["workflow-list"].querySelectorAll("button")].forEach((button) => { + button.setAttribute("aria-selected", String(button.dataset.workflowId === id)); + }); +} + +export async function loadWorkflows() { + if (!state.projectId) return; + elements["workflow-status"].textContent = "Loading workflow registry…"; + const payload = await api(base()); + state.workflows = payload.workflows ?? []; + elements["workflow-list"].replaceChildren( + ...state.workflows.map((workflow) => { + const button = document.createElement("button"); + button.type = "button"; + button.dataset.workflowId = workflow.workflow_id; + button.textContent = `${workflow.workflow_id} · r${workflow.latest_revision ?? "—"}${workflow.active ? " · active" : ""}`; + button.addEventListener("click", () => selectWorkflow(workflow.workflow_id).catch(showError)); + return button; + }), + ); + elements["workflow-empty"].hidden = state.workflows.length > 0; + if (state.workflows[0]) await selectWorkflow(state.workflowId ?? state.workflows[0].workflow_id); + elements["workflow-status"].textContent = + "Registry loaded. Drafts cannot change while a run is active."; + updateMutationControls(); +} + +export function bindWorkflowEditor() { + elements["workflow-refresh"].addEventListener("click", () => loadWorkflows().catch(showError)); + elements["workflow-draft-form"].addEventListener("submit", async (event) => { + event.preventDefault(); + try { + const workflow = JSON.parse(elements["workflow-definition"].value); + const result = await api(`${base()}/${encodeURIComponent(state.workflowId)}/drafts`, { + method: "POST", + body: JSON.stringify({ + workflow, + expected_revision: selectedRevision(), + actor: elements["workflow-actor"].value, + rationale: elements["workflow-rationale"].value, + }), + }); + state.workflow.revisions.push(result.revision); + state.workflow.workflow = result.revision.workflow; + state.workflow.digest = result.revision.digest; + showToast("Draft revision saved.", "notice"); + renderWorkflow(); + } catch (error) { + showError(error); + } + }); + elements["workflow-validate"].addEventListener("click", async () => { + try { + const revision = selectedRevision(); + const result = await api( + `${base()}/${encodeURIComponent(state.workflowId)}/revisions/${encodeURIComponent(revision)}/validate`, + { method: "POST", body: "{}" }, + ); + elements["workflow-status"].textContent = `Validation: ${JSON.stringify(result.validation)}`; + } catch (error) { + showError(error); + } + }); + elements["workflow-diff"].addEventListener("click", async () => { + try { + const diff = await api( + `${base()}/${encodeURIComponent(state.workflowId)}/diff?from=${encodeURIComponent(elements["workflow-diff-base"].value)}&to=${encodeURIComponent(selectedRevision())}`, + ); + elements["workflow-diff-output"].textContent = JSON.stringify(diff.diff, null, 2); + } catch (error) { + showError(error); + } + }); + elements["workflow-activate"].addEventListener("click", async () => { + try { + const revision = selectedRevision(); + const result = await api( + `${base()}/${encodeURIComponent(state.workflowId)}/revisions/${encodeURIComponent(revision)}/activate`, + { + method: "POST", + body: JSON.stringify({ + digest: elements["workflow-digest-confirmation"].value, + actor: elements["workflow-actor"].value, + rationale: elements["workflow-rationale"].value, + }), + }, + ); + elements["workflow-status"].textContent = + `Activated ${result.activation?.revision ?? revision}.`; + await selectWorkflow(state.workflowId); + } catch (error) { + showError(error); + } + }); +} diff --git a/packages/orchestration/operator/static/styles.css b/packages/orchestration/operator/static/styles.css index 75af8f1..1823ea7 100644 --- a/packages/orchestration/operator/static/styles.css +++ b/packages/orchestration/operator/static/styles.css @@ -11,3 +11,4 @@ @import url("./css/evidence.css"); @import url("./css/dialogs.css"); @import url("./css/responsive.css"); +@import url("./css/workflows.css"); diff --git a/packages/orchestration/operator/tests/pages-demo.test.mjs b/packages/orchestration/operator/tests/pages-demo.test.mjs new file mode 100644 index 0000000..035d4a1 --- /dev/null +++ b/packages/orchestration/operator/tests/pages-demo.test.mjs @@ -0,0 +1,70 @@ +/** Verifies the Pages artifact stays derived, sanitized, and entirely simulated. */ + +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { mkdtempSync, readFileSync, readdirSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import test from "node:test"; + +const orchestrationRoot = resolve(import.meta.dirname, "../.."); +const staticRoot = resolve(orchestrationRoot, "operator/static"); +const buildScript = resolve(orchestrationRoot, "scripts/build-pages-demo.sh"); + +function buildDemo(t) { + const output = mkdtempSync(join(tmpdir(), "rae-pages-demo-")); + t.after(() => rmSync(output, { recursive: true, force: true })); + execFileSync("bash", [buildScript, "--output", output]); + return output; +} + +test("Pages demo copies canonical application modules without forking the product UI", (t) => { + const output = buildDemo(t); + assert.deepEqual(readdirSync(resolve(output, "js")), readdirSync(resolve(staticRoot, "js"))); + for (const name of readdirSync(resolve(staticRoot, "js"))) { + assert.equal( + readFileSync(resolve(output, "js", name), "utf8"), + readFileSync(resolve(staticRoot, "js", name), "utf8"), + ); + } + assert.equal( + readFileSync(resolve(output, "app.js"), "utf8"), + readFileSync(resolve(staticRoot, "app.js"), "utf8"), + ); +}); + +test("Pages demo is base-path safe and labels its evidence and actions as simulated", (t) => { + const output = buildDemo(t); + const html = readFileSync(resolve(output, "index.html"), "utf8"); + const mock = readFileSync(resolve(output, "demo/mock-api.js"), "utf8"); + assert.match(html, /Static simulation/); + assert.match(html, /Sanitized fixture data\. No command is run and no state is saved\./); + assert.match(html, /href="\.\/styles\.css"/); + assert.match(html, /src="\.\/demo\/mock-api\.js"/); + assert.doesNotMatch(html, /(?:href|src)="\//); + for (const id of [ + "new-run-button", + "start-submit", + "stop-button", + "interrupt-button", + "resume-button", + "cleanup-button", + "confirm-submit", + ]) { + assert.ok(mock.includes(`"${id}"`)); + } + assert.match(mock, /document\.querySelectorAll\("\[data-decision\]"\)/); + assert.match(mock, /window\.fetch = demoFetch/); + assert.match(mock, /does not permit network requests/); +}); + +test("Pages fixtures contain no local path, credential, provider, or raw prompt data", (t) => { + const output = buildDemo(t); + const mock = readFileSync(resolve(output, "demo/mock-api.js"), "utf8"); + assert.doesNotMatch( + mock, + /\/Users\/|\/home\/|Bearer |api[_-]?key|private-provider|must-not-leak/i, + ); + assert.match(mock, /sebastianspicker\/rae · fixture/); + assert.match(mock, /\.git\/rae-worktrees\//); +}); diff --git a/packages/orchestration/operator/tests/runs.test.mjs b/packages/orchestration/operator/tests/runs.test.mjs index 24e67fd..711adc8 100644 --- a/packages/orchestration/operator/tests/runs.test.mjs +++ b/packages/orchestration/operator/tests/runs.test.mjs @@ -68,6 +68,15 @@ test("durable discovery projects run state without exposing raw trace metadata", assert.equal(runs[0].task, "Verify projected events"); assert.equal(runs[0].current_phase, "quality-tests"); assert.equal(publicRun(runs[0]).workspaceRoot, undefined); + assert.deepEqual(runs[0].graph_health, { + available: false, + valid: false, + node_count: 0, + edge_count: 0, + stale_sources: 0, + stale_memory: 0, + unresolved_conflicts: 0, + }); const page = paginatedEvents(runs[0], { after: 0, limit: 10 }); assert.equal(page.events.length, 1); assert.equal(page.events[0].event, "agent_call"); @@ -76,6 +85,57 @@ test("durable discovery projects run state without exposing raw trace metadata", assert.doesNotMatch(JSON.stringify(page), /must-not-leak|private-provider/); }); +test("graph workflow discovery exposes sanitized node-instance progress without provider payloads", () => { + const { root, runId } = createDiscoverableRun(); + const runDir = join(root, ".pipeline", "runs", runId); + writeJson(join(runDir, "request.json"), { + schema_version: "2.1.0", + task: "Map modules", + requested_at: "2026-07-17T09:42:00.000Z", + workflow: { + mode: "graph-native", + workflow_id: "module-map", + revision: 2, + digest: "a".repeat(64), + snapshot: { schema_version: "2.1.0", budgets: { max_concurrency: 4 } }, + }, + }); + const attempts = join(runDir, "workflow", "attempts", "inspect"); + mkdirSync(attempts, { recursive: true }); + writeJson(join(attempts, "inspect_item.1.json"), { + schema_version: "2.1.0", + workflow_digest: "a".repeat(64), + node_id: "inspect", + instance_id: "inspect:item-a", + parent_node: "discover", + item_key: "module-a", + item_digest: "b".repeat(64), + status: "passed", + attempt: 1, + execution_tier: "judgment", + payload: { secret_provider_payload: "must-not-leak" }, + resource_usage: { tokens: 1234 }, + quorum: { threshold: 1, passed: 1 }, + }); + + const [run] = discoverRuns({ id: "project_12345678", root, label: root }); + assert.equal(run.workflow.workflow_id, "module-map"); + assert.deepEqual(run.workflow.instances[0], { + instance_id: "inspect:item-a", + node_id: "inspect", + parent_node: "discover", + item_key: "module-a", + item_digest: "b".repeat(64), + status: "passed", + attempt: 1, + execution_tier: "judgment", + selection: null, + quorum: { threshold: 1, passed: 1 }, + convergence: null, + }); + assert.doesNotMatch(JSON.stringify(publicRun(run)), /must-not-leak|tokens/); +}); + test("active guard discovery returns phase-active without consuming poisoned pipeline state", (t) => { const { root, runId } = createDiscoverableRun(); t.after(() => { diff --git a/packages/orchestration/operator/tests/server.test.mjs b/packages/orchestration/operator/tests/server.test.mjs index 34663d3..bab1169 100644 --- a/packages/orchestration/operator/tests/server.test.mjs +++ b/packages/orchestration/operator/tests/server.test.mjs @@ -53,14 +53,14 @@ const host = "127.0.0.1:4173"; const origin = `http://${host}`; const project = { id: "project_12345678", root: "/tmp", label: "/tmp" }; -function requestContext(controller) { - return { token, host, origin, projects: [project], controller }; +function requestContext(controller, workflowRegistry) { + return { token, host, origin, projects: [project], controller, workflowRegistry }; } -async function dispatch(options, controller) { +async function dispatch(options, controller, workflowRegistry) { const req = new MockRequest(options); const res = new MockResponse(); - await handleOperatorRequest(req, res, requestContext(controller)); + await handleOperatorRequest(req, res, requestContext(controller, workflowRegistry)); return res; } @@ -159,3 +159,59 @@ test("API rejects bodies declared above 64KiB before control dispatch", async () assert.equal(res.statusCode, 413); assert.equal(called, false); }); + +test("workflow editor API keeps registry mutations authenticated and rejects active-run edits", async () => { + const calls = []; + const registry = { + list: () => [{ id: "release", active_revision: "3" }], + show: (id) => ({ id, revisions: ["2", "3"] }), + draft: (id, body) => { + calls.push(["draft", id, body]); + return { id: "4", status: "draft" }; + }, + validate: (id, revision) => ({ id, revision, valid: true }), + diff: (id, query) => ({ id, ...query, changes: [] }), + activate: (id, revision) => ({ id, revision, activated: true }), + }; + const controller = { ownedRunId: null, refreshOwnership: () => null }; + const list = await dispatch( + { + path: `/api/v1/projects/${project.id}/workflows`, + headers: { host, authorization: `Bearer ${token}` }, + }, + controller, + registry, + ); + assert.equal(list.statusCode, 200); + assert.equal(JSON.parse(list.body).workflows[0].id, "release"); + const body = JSON.stringify({ base_revision: "3", definition: { stages: [] } }); + const draft = await dispatch( + { + path: `/api/v1/projects/${project.id}/workflows/release/drafts`, + method: "POST", + body, + headers: { + host, + origin, + authorization: `Bearer ${token}`, + "content-length": Buffer.byteLength(body), + }, + }, + controller, + registry, + ); + assert.equal(draft.statusCode, 201); + assert.deepEqual(calls[0][2], JSON.parse(body)); + controller.ownedRunId = "run_12345678"; + const blocked = await dispatch( + { + path: `/api/v1/projects/${project.id}/workflows/release/revisions/4/activate`, + method: "POST", + body: "{}", + headers: { host, origin, authorization: `Bearer ${token}`, "content-length": "2" }, + }, + controller, + registry, + ); + assert.equal(blocked.statusCode, 409); +}); diff --git a/packages/orchestration/operator/tests/ui-contract.test.mjs b/packages/orchestration/operator/tests/ui-contract.test.mjs index 29baccc..1d3f3b1 100644 --- a/packages/orchestration/operator/tests/ui-contract.test.mjs +++ b/packages/orchestration/operator/tests/ui-contract.test.mjs @@ -168,3 +168,17 @@ test("static styles and scripts are modularized under static/css and static/js", assert.match(appEntry, /from "\.\/js\/handlers\.js"/); assert.match(appEntry, /from "\.\/js\/data\.js"/); }); + +test("workflow map provides topology, typed edges, budgets, and accessible instance state", () => { + assert.match(app, /function topology\(definition\)/); + assert.match(app, /workflow-edge--\$\{edgeRecord\.type\}/); + assert.match(app, /fanout:/); + assert.match(app, /Live and completed node instances/); + assert.match(app, /execution_tier/); + assert.match(app, /quorum/); + assert.match(app, /max_dynamic_instances/); + assert.match(app, /Registry is read-only while a run is active/); + assert.match(css, /\.workflow-edge--stream/); + assert.match(css, /\.workflow-edge--loop-back/); + assert.match(html, /complete equivalent structure and live instance state/); +}); diff --git a/packages/orchestration/package.json b/packages/orchestration/package.json index 16c60ae..7a05baf 100644 --- a/packages/orchestration/package.json +++ b/packages/orchestration/package.json @@ -15,6 +15,9 @@ }, "scripts": { "agent": "node scripts/pipeline/autonomous.mjs", + "benchmark:graph-context": "node scripts/eval/graph-context-benchmark.mjs", + "build:pages-demo": "bash scripts/build-pages-demo.sh", + "benchmark:workflow-topology": "node scripts/eval/workflow-topology-benchmark.mjs", "build": "npm run build --workspaces --if-present", "test:operator": "node --test operator/tests/*.test.mjs", "test:runner": "cd scripts/pipeline && ../../node_modules/.bin/vitest run", diff --git a/packages/orchestration/scripts/build-pages-demo.sh b/packages/orchestration/scripts/build-pages-demo.sh new file mode 100644 index 0000000..2a3c7f8 --- /dev/null +++ b/packages/orchestration/scripts/build-pages-demo.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# Builds the GitHub Pages demo from the canonical operator UI plus the fixture adapter. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ORCHESTRATION_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +REPOSITORY_ROOT="$(cd "$ORCHESTRATION_ROOT/../.." && pwd)" +STATIC_ROOT="$ORCHESTRATION_ROOT/operator/static" +DEMO_ROOT="$ORCHESTRATION_ROOT/operator/demo" +OUTPUT_ROOT="$REPOSITORY_ROOT/dist/pages-demo" + +if [[ "${1:-}" == "--output" ]]; then + [[ -n "${2:-}" && $# -eq 2 ]] || { + echo "--output requires exactly one directory" >&2 + exit 2 + } + case "$2" in + /*) OUTPUT_ROOT="$2" ;; + *) OUTPUT_ROOT="$PWD/$2" ;; + esac +elif [[ $# -ne 0 ]]; then + echo "usage: build-pages-demo.sh [--output directory]" >&2 + exit 2 +fi + +[[ -n "$OUTPUT_ROOT" && "$OUTPUT_ROOT" != "/" ]] || { + echo "refusing unsafe output directory" >&2 + exit 2 +} + +rm -rf -- "$OUTPUT_ROOT" +mkdir -p "$OUTPUT_ROOT/demo" +cp -R "$STATIC_ROOT/." "$OUTPUT_ROOT/" +cp "$DEMO_ROOT/mock-api.js" "$OUTPUT_ROOT/demo/mock-api.js" +cp "$DEMO_ROOT/demo.css.txt" "$OUTPUT_ROOT/demo/demo.css" + +INDEX="$OUTPUT_ROOT/index.html" + +replace_once() { + local needle="$1" + local replacement="$2" + local count + count="$(NEEDLE="$needle" perl -0777 -ne '$count = () = /\Q$ENV{NEEDLE}\E/g; print $count' "$INDEX")" + [[ "$count" == "1" ]] || { + echo "expected exactly one canonical HTML anchor: $needle" >&2 + exit 1 + } + NEEDLE="$needle" REPLACEMENT="$replacement" perl -0777 -i -pe \ + 'BEGIN { $needle = $ENV{NEEDLE}; $replacement = $ENV{REPLACEMENT} } s/\Q$needle\E/$replacement/' \ + "$INDEX" +} + +replace_once \ + "RAE Evidence Dossier — local Runboard operator console for autonomous repository runs." \ + "RAE Evidence Dossier static simulation using sanitized fixture data. No command is run." +replace_once \ + "RAE Evidence Dossier" \ + "RAE Evidence Dossier · Static simulation" +replace_once \ + '' \ + $'\n ' +replace_once \ + '' \ + '' +replace_once \ + " " \ + $' \n ' + +touch "$OUTPUT_ROOT/.nojekyll" +echo "Built static demo at $OUTPUT_ROOT" diff --git a/packages/orchestration/scripts/eval/aggregate.mjs b/packages/orchestration/scripts/eval/aggregate.mjs index d0431c6..18b8201 100755 --- a/packages/orchestration/scripts/eval/aggregate.mjs +++ b/packages/orchestration/scripts/eval/aggregate.mjs @@ -2,8 +2,7 @@ /** * Aggregates pipeline evaluation artifacts into comparable configuration metrics for repeatable analysis. */ -import { existsSync, mkdirSync, writeFileSync } from "node:fs"; -import { dirname, resolve } from "node:path"; +import { resolve } from "node:path"; import { pathToFileURL } from "node:url"; import { parseArgs as parseCliArgs } from "../lib/argv.mjs"; import { CONFIG_IDS } from "../lib/constants.mjs"; @@ -12,6 +11,7 @@ import { readJson, resolveWithinRepo, toWorkspaceRelative, + writeJson, } from "../pipeline/lib/state.mjs"; import { assertSupportedNodeRuntime } from "../lib/node-runtime.mjs"; @@ -55,7 +55,7 @@ function p95(values) { if (values.length === 0) return 0; const sorted = [...values].sort((a, b) => a - b); const idx = Math.min(sorted.length - 1, Math.ceil(sorted.length * 0.95) - 1); - return sorted[idx]; + return sorted.at(idx); } function driftScoreFromClaims(claims) { @@ -84,17 +84,7 @@ export function gatePassRate(gates) { ); } -function loadRunMetrics(root, runId) { - const runDir = getRunDir(runId, root); - if (!existsSync(runDir)) { - throw new Error(`run directory not found for run_id=${runId}`); - } - const gatesDir = resolve(runDir, "gates"); - const traceSummary = readJson(resolve(runDir, "trace.summary.json"), {}); - const drift = readJson(resolve(runDir, "drift-reports", "pmatch.json"), {}); - const review = readJson(resolve(runDir, "review.json"), {}); - const security = readJson(resolve(runDir, "quality-reports", "security.json"), {}); - +function loadGateResults(gatesDir) { const gateFiles = [ "arm-gate.json", "design-gate.json", @@ -107,22 +97,37 @@ function loadRunMetrics(root, runId) { "postbuild-gate.json", "release-readiness-gate.json", ]; - const gateResults = gateFiles + return gateFiles .map((name) => readJson(resolve(gatesDir, name), null)) .filter(Boolean) - .map((g) => ({ phase: g.phase, status: g.status })); + .map((gate) => ({ phase: gate.phase, status: gate.status })); +} - const success = gateResults.length > 0 && gateResults.every((g) => g.status === "pass"); +function reviewMetrics(review) { const rawFindings = Array.isArray(review.reviewers) ? review.reviewers.reduce( - (acc, r) => acc + (Array.isArray(r.findings) ? r.findings.length : 0), + (total, reviewer) => + total + (Array.isArray(reviewer.findings) ? reviewer.findings.length : 0), 0, ) : 0; - const dedupFindings = Array.isArray(review.deduplicated_findings) + const deduplicatedFindings = Array.isArray(review.deduplicated_findings) ? review.deduplicated_findings.length : 0; - const dedupRatio = dedupFindings > 0 ? rawFindings / dedupFindings : 1; + return { dedupRatio: deduplicatedFindings > 0 ? rawFindings / deduplicatedFindings : 1 }; +} + +function loadRunMetrics(root, runId) { + const runDir = getRunDir(runId, root); + const gatesDir = resolve(runDir, "gates"); + const traceSummary = readJson(resolve(runDir, "trace.summary.json"), null); + if (!traceSummary) throw new Error(`run directory not found for run_id=${runId}`); + const drift = readJson(resolve(runDir, "drift-reports", "pmatch.json"), {}); + const review = readJson(resolve(runDir, "review.json"), {}); + const security = readJson(resolve(runDir, "quality-reports", "security.json"), {}); + const gateResults = loadGateResults(gatesDir); + const success = gateResults.length > 0 && gateResults.every((g) => g.status === "pass"); + const { dedupRatio } = reviewMetrics(review); return { run_id: runId, @@ -223,8 +228,7 @@ function main() { }; const outPath = resolveWithinRepo(output, root); - mkdirSync(dirname(outPath), { recursive: true }); - writeFileSync(outPath, `${JSON.stringify(report, null, 2)}\n`, "utf8"); + writeJson(outPath, report); process.stdout.write(`${outPath}\n`); } diff --git a/packages/orchestration/scripts/eval/drift-benchmark.mjs b/packages/orchestration/scripts/eval/drift-benchmark.mjs index 17e68fe..5a237a1 100755 --- a/packages/orchestration/scripts/eval/drift-benchmark.mjs +++ b/packages/orchestration/scripts/eval/drift-benchmark.mjs @@ -5,15 +5,14 @@ import { existsSync, mkdirSync, - mkdtempSync, - readFileSync, + mkdtempSync as createTempDirectory, readdirSync, - rmSync, + rmSync as removeTempDirectory, writeFileSync, } from "node:fs"; -import { dirname, join, relative, resolve } from "node:path"; +import { join, relative, resolve } from "node:path"; import { spawnSync } from "node:child_process"; -import { resolveWithinRepo } from "../pipeline/lib/state.mjs"; +import { readJsonStrict, resolveWithinRepo, writeJson } from "../pipeline/lib/state.mjs"; import { parseArgs as parseCliArgs } from "../lib/argv.mjs"; import { assertSupportedNodeRuntime } from "../lib/node-runtime.mjs"; @@ -73,7 +72,7 @@ function toMetrics(tp, fp, fn) { } function evaluateByClass(expected, predicted) { - const byClass = {}; + const byClass = new Map(); let totalTp = 0; let totalFp = 0; let totalFn = 0; @@ -85,7 +84,7 @@ function evaluateByClass(expected, predicted) { const fp = Math.max(0, predictedCount - expectedCount); const fn = Math.max(0, expectedCount - predictedCount); - byClass[cls] = toMetrics(tp, fp, fn); + byClass.set(cls, toMetrics(tp, fp, fn)); totalTp += tp; totalFp += fp; totalFn += fn; @@ -93,65 +92,68 @@ function evaluateByClass(expected, predicted) { return { overall: toMetrics(totalTp, totalFp, totalFn), - by_class: byClass, + byClass, }; } -function runSkill(repoRoot, fixture, targetRef, mode) { - const skillEntrypoint = resolve(repoRoot, "skills/dev-tools/multi-model-review/dist/index.js"); - if (!existsSync(skillEntrypoint)) { - throw new Error( - "multi-model-review dist/index.js not found. Run npm run build in skills/dev-tools/multi-model-review first.", - ); +function driftInput(fixture, targetRef, mode) { + const driftConfig = { target_ref: targetRef, mode }; + if (mode !== "dual-extractor") { + return { + action: { type: "drift-detect" }, + document: { content: fixture.source, type: "plan" }, + drift_config: driftConfig, + }; } - - const driftConfig = { - target_ref: targetRef, - mode, - }; - - if (mode === "dual-extractor") { - if (!Array.isArray(fixture.extractor_claim_sets) || fixture.extractor_claim_sets.length !== 2) { - throw new Error(`fixture ${fixture.id} missing extractor_claim_sets for dual-extractor mode`); - } - driftConfig.extractor_claim_sets = fixture.extractor_claim_sets; + if (!Array.isArray(fixture.extractor_claim_sets) || fixture.extractor_claim_sets.length !== 2) { + throw new Error(`fixture ${fixture.id} missing extractor_claim_sets for dual-extractor mode`); } - - const input = { + return { action: { type: "drift-detect" }, document: { content: fixture.source, type: "plan" }, - drift_config: driftConfig, + drift_config: { ...driftConfig, extractor_claim_sets: fixture.extractor_claim_sets }, }; +} +function runDriftSkill(skillEntrypoint, repoRoot, input) { const result = spawnSync("node", [skillEntrypoint], { cwd: repoRoot, input: JSON.stringify(input), encoding: "utf8", - env: { - ...process.env, - WORKSPACE_ROOT: repoRoot, - }, + env: { ...process.env, WORKSPACE_ROOT: repoRoot }, }); + if (result.error) throw result.error; + if (!result.stdout && !result.stderr) throw new Error("drift-detect returned empty output"); + if (result.status !== 0) throw new Error(result.stderr || result.stdout || "drift-detect failed"); + return result.stdout; +} - const rawOut = result.stdout || result.stderr; - if (!rawOut) { - throw new Error("drift-detect returned empty output"); - } - if (result.status !== 0) { - throw new Error(result.stderr || result.stdout || "drift-detect failed"); - } - +function parseDriftResult(output) { let parsed; try { - parsed = JSON.parse(result.stdout); + parsed = JSON.parse(output); } catch (error) { throw new Error(`drift-detect returned invalid JSON: ${String(error)}`); } - if (!parsed.success) { - throw new Error(parsed.error?.message || "drift-detect failed"); + if (!parsed.success) throw new Error(parsed.error?.message || "drift-detect failed"); + return parsed.data; +} + +function runSkill(repoRoot, fixture, targetRef, mode) { + if ( + !withWorkingDirectory(repoRoot, () => + existsSync("skills/dev-tools/multi-model-review/dist/index.js"), + ) + ) { + throw new Error( + "multi-model-review dist/index.js not found. Run npm run build in skills/dev-tools/multi-model-review first.", + ); } - return parsed.data; + const skillEntrypoint = resolve(repoRoot, "skills/dev-tools/multi-model-review/dist/index.js"); + return parseDriftResult( + runDriftSkill(skillEntrypoint, repoRoot, driftInput(fixture, targetRef, mode)), + ); } function normalizeExpected(fixture) { @@ -184,8 +186,8 @@ function requireFixtureObject(fixture, fileName) { } } -function requireFixtureText(fixture, field) { - if (typeof fixture[field] !== "string" || fixture[field].length === 0) { +function requireFixtureText(fixture, field, value) { + if (typeof value !== "string" || value.length === 0) { throw new Error(`fixture ${fixture.id} is missing non-empty ${field}`); } } @@ -197,119 +199,159 @@ function validateFixtureShape(fixture, fileName) { `fixture ${fileName} has invalid id: must match ^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$`, ); } - requireFixtureText(fixture, "source"); - requireFixtureText(fixture, "target"); + requireFixtureText(fixture, "source", fixture.source); + requireFixtureText(fixture, "target", fixture.target); } -function main() { - const args = parseArgs(process.argv); +function benchmarkPaths(args) { const repoRoot = resolve(args.root); - const casesDir = resolve(repoRoot, "docs/eval/drift_goldset/cases"); - const outPath = resolveWithinRepo(args.output, repoRoot); - const workspaceTmpRoot = resolveWithinRepo(".pipeline/tmp", repoRoot); - - const thresholds = { - precision: args.precisionMin, - recall: args.recallMin, - f1: args.f1Min, + return { + repoRoot, + casesDir: resolveWithinRepo("docs/eval/drift_goldset/cases", repoRoot), + outPath: resolveWithinRepo(args.output, repoRoot), + workspaceTmpRoot: resolveWithinRepo(".pipeline/tmp", repoRoot), + thresholds: { + precision: args.precisionMin, + recall: args.recallMin, + f1: args.f1Min, + }, }; +} - mkdirSync(workspaceTmpRoot, { recursive: true }); - mkdirSync(dirname(outPath), { recursive: true }); - const tmpDir = mkdtempSync(join(workspaceTmpRoot, "drift-benchmark-")); +function modeResultStore() { + return new Map(MODES.map((mode) => [mode, { expected: [], predicted: [], cases: [] }])); +} - const files = readdirSync(casesDir) +function withWorkingDirectory(directory, action) { + const original = process.cwd(); + process.chdir(directory); + try { + return action(); + } finally { + process.chdir(original); + } +} + +function fixtureFiles(casesDir) { + return withWorkingDirectory(casesDir, () => readdirSync(".")) .filter((file) => file.endsWith(".json")) .sort(); +} - const modeResults = {}; - for (const mode of MODES) { - modeResults[mode] = { - expected: [], - predicted: [], - cases: [], - }; - } +function writeFixtureTarget(tmpDir, target) { + withWorkingDirectory(tmpDir, () => writeFileSync("target.md", target, "utf8")); + return join(tmpDir, "target.md"); +} - try { - for (const file of files) { - const fixturePath = resolve(casesDir, file); - const fixture = JSON.parse(readFileSync(fixturePath, "utf8")); - validateFixtureShape(fixture, file); - const targetPath = join(tmpDir, `${fixture.id}.target.md`); - writeFileSync(targetPath, fixture.target, "utf8"); - const targetRef = relative(repoRoot, targetPath); - const expected = normalizeExpected(fixture); - - for (const mode of MODES) { - const drift = runSkill(repoRoot, fixture, targetRef, mode); - const predicted = normalizePredicted(drift); - const metrics = evaluateByClass(expected, predicted); - - modeResults[mode].expected.push(...expected); - modeResults[mode].predicted.push(...predicted); - modeResults[mode].cases.push({ - case_id: fixture.id, - metrics, - expected, - predicted, - }); - } +function collectFixtureResults({ files, casesDir, modeResults, repoRoot, tmpDir }) { + for (const file of files) { + const fixturePath = resolve(casesDir, file); + const fixture = readJsonStrict(fixturePath); + validateFixtureShape(fixture, file); + const targetPath = writeFixtureTarget(tmpDir, fixture.target); + const targetRef = relative(repoRoot, targetPath); + const expected = normalizeExpected(fixture); + + for (const mode of MODES) { + const drift = runSkill(repoRoot, fixture, targetRef, mode); + const predicted = normalizePredicted(drift); + const metrics = evaluateByClass(expected, predicted); + const result = modeResults.get(mode); + + result.expected.push(...expected); + result.predicted.push(...predicted); + result.cases.push({ + case_id: fixture.id, + metrics: { overall: metrics.overall, by_class: Object.fromEntries(metrics.byClass) }, + expected, + predicted, + }); } + } +} - const metricsByMode = {}; - const metricsByClass = {}; +function aggregateModeResults(modeResults) { + const metricsByMode = new Map(); + const metricsByClass = new Map(TAXONOMY.map((cls) => [cls, new Map()])); + for (const mode of MODES) { + const result = modeResults.get(mode); + const aggregate = evaluateByClass(result.expected, result.predicted); + metricsByMode.set(mode, aggregate.overall); for (const cls of TAXONOMY) { - metricsByClass[cls] = {}; + metricsByClass.get(cls).set(mode, aggregate.byClass.get(cls)); } - - for (const mode of MODES) { - const aggregate = evaluateByClass(modeResults[mode].expected, modeResults[mode].predicted); - metricsByMode[mode] = aggregate.overall; - for (const cls of TAXONOMY) { - metricsByClass[cls][mode] = aggregate.by_class[cls]; - } - modeResults[mode].aggregate = aggregate; - } - - const overall = { - precision: MODES.reduce((acc, mode) => acc + metricsByMode[mode].precision, 0) / MODES.length, - recall: MODES.reduce((acc, mode) => acc + metricsByMode[mode].recall, 0) / MODES.length, - f1: MODES.reduce((acc, mode) => acc + metricsByMode[mode].f1, 0) / MODES.length, + result.aggregate = { + overall: aggregate.overall, + by_class: Object.fromEntries(aggregate.byClass), }; + } + return { metricsByMode, metricsByClass }; +} + +function overallMetrics(metricsByMode) { + const meanMetric = (metric) => + MODES.reduce((sum, mode) => sum + metric(metricsByMode.get(mode)), 0) / MODES.length; + return { + precision: meanMetric((metrics) => metrics.precision), + recall: meanMetric((metrics) => metrics.recall), + f1: meanMetric((metrics) => metrics.f1), + }; +} - const failedModes = MODES.filter((mode) => thresholdFailed(metricsByMode[mode], thresholds)); - - const report = { - generated_at: new Date().toISOString(), - case_count: files.length, - thresholds, - metrics_by_mode: metricsByMode, - metrics_by_class: metricsByClass, - overall, - modes: Object.fromEntries( - MODES.map((mode) => [ +function benchmarkReport({ files, thresholds, metricsByMode, metricsByClass, modeResults }) { + const failedModes = MODES.filter((mode) => thresholdFailed(metricsByMode.get(mode), thresholds)); + return { + generated_at: new Date().toISOString(), + case_count: files.length, + thresholds, + metrics_by_mode: Object.fromEntries(metricsByMode), + metrics_by_class: Object.fromEntries( + [...metricsByClass].map(([cls, metrics]) => [cls, Object.fromEntries(metrics)]), + ), + overall: overallMetrics(metricsByMode), + modes: Object.fromEntries( + MODES.map((mode) => { + const aggregate = modeResults.get(mode).aggregate; + return [ mode, { - overall: modeResults[mode].aggregate.overall, - by_class: modeResults[mode].aggregate.by_class, - cases: modeResults[mode].cases, + overall: aggregate.overall, + by_class: aggregate.by_class, + cases: modeResults.get(mode).cases, }, - ]), - ), - status: failedModes.length === 0 ? "pass" : "fail", - failed_modes: failedModes, - }; + ]; + }), + ), + status: failedModes.length === 0 ? "pass" : "fail", + failed_modes: failedModes, + }; +} - writeFileSync(outPath, `${JSON.stringify(report, null, 2)}\n`, "utf8"); - process.stdout.write(`${outPath}\n`); +function main() { + const paths = benchmarkPaths(parseArgs(process.argv)); + withWorkingDirectory(paths.repoRoot, () => mkdirSync(".pipeline/tmp", { recursive: true })); + const tmpDir = createTempDirectory(join(paths.workspaceTmpRoot, "drift-benchmark-")); + const files = fixtureFiles(paths.casesDir); + const modeResults = modeResultStore(); - if (failedModes.length > 0) { + try { + collectFixtureResults({ ...paths, files, modeResults, tmpDir }); + const { metricsByMode, metricsByClass } = aggregateModeResults(modeResults); + const report = benchmarkReport({ + ...paths, + files, + metricsByMode, + metricsByClass, + modeResults, + }); + writeJson(paths.outPath, report); + process.stdout.write(`${paths.outPath}\n`); + if (report.failed_modes.length > 0) { process.exitCode = 1; } } finally { - rmSync(tmpDir, { recursive: true, force: true }); + removeTempDirectory(tmpDir, { recursive: true, force: true }); } } diff --git a/packages/orchestration/scripts/eval/graph-context-benchmark.mjs b/packages/orchestration/scripts/eval/graph-context-benchmark.mjs new file mode 100644 index 0000000..df291ba --- /dev/null +++ b/packages/orchestration/scripts/eval/graph-context-benchmark.mjs @@ -0,0 +1,221 @@ +#!/usr/bin/env node +/** Compares frozen flat, lexical, graph, and graph-memory repository context retrieval. */ +import { performance } from "node:perf_hooks"; +import { resolve } from "node:path"; +import { + loadGraph, + projectGraph, + queryGraph, + retrieveMemoryContext, +} from "../pipeline/lib/graph.mjs"; +import { + projectSourcePath, + readUtf8RegularFile, + writePrivateUtf8File, +} from "../pipeline/lib/graph/core.mjs"; + +function parse(argv) { + const options = { dataset: undefined, json: false, output: undefined, projectRoot: undefined }; + const remaining = [...argv]; + while (remaining.length > 0) { + const token = remaining.shift(); + if (assignBooleanOption(options, token)) continue; + if (!token.startsWith("--")) throw new Error(`unexpected argument: ${token}`); + const value = remaining.shift(); + if (!value || value.startsWith("--")) throw new Error(`missing value for ${token}`); + assignValueOption(options, token, value); + } + return options; +} + +function assignBooleanOption(options, option) { + switch (option) { + case "--json": + options.json = true; + return true; + default: + return false; + } +} + +function assignValueOption(options, option, value) { + switch (option) { + case "--dataset": + options.dataset = value; + return; + case "--output": + options.output = value; + return; + case "--project-root": + options.projectRoot = value; + return; + default: + throw new Error(`unexpected argument: ${option}`); + } +} + +function tokens(value) { + return new Set( + String(value) + .toLowerCase() + .match(/[a-z0-9_./-]{2,}/g) ?? [], + ); +} + +function overlap(query, value) { + const expected = tokens(query); + const actual = tokens(value); + return [...expected].filter((token) => actual.has(token)).length; +} + +function percentile(values, quantile) { + if (!values.length) return 0; + const sorted = [...values].sort((a, b) => a - b); + return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * quantile) - 1)]; +} + +function flatRank(files, query, includeContent) { + return files + .map((file) => ({ + path: file.attributes.path, + snippet: includeContent ? file.snippet : file.attributes.path, + score: overlap(query, `${file.attributes.path} ${includeContent ? file.snippet : ""}`), + })) + .filter((item) => item.score > 0) + .sort((a, b) => b.score - a.score || a.path.localeCompare(b.path)) + .slice(0, 10); +} + +function graphRank(projectRoot, runId, task, includeMemory) { + const bundle = queryGraph({ + projectRoot, + runId, + seed: task.query, + phase: `benchmark-${task.id}`, + maxRecords: 10, + }); + const records = bundle.records + .filter((record) => record.kind === "File") + .map((record) => ({ path: record.source_ref, snippet: record.snippet })); + if (includeMemory) retrieveMemoryContext({ projectRoot, seed: task.query, limit: 10 }); + return records.slice(0, 10); +} + +function evaluateMode(mode, tasks, retrieve) { + let hits = 0; + let contextTokens = 0; + let stale = 0; + let records = 0; + const latencies = []; + const taskResults = []; + for (const task of tasks) { + const started = performance.now(); + const selected = retrieve(task); + latencies.push(performance.now() - started); + const paths = selected.map((item) => item.path); + const hit = task.expected_paths.some((path) => paths.includes(path)); + if (hit) hits++; + contextTokens += Math.ceil( + selected.reduce((total, item) => total + String(item.snippet ?? item.path).length, 0) / 4, + ); + stale += selected.filter((item) => item.staleness === "stale").length; + records += selected.length; + taskResults.push({ task_id: task.id, hit_at_10: hit, selected_paths: paths }); + } + return { + mode, + file_localization_recall_at_10: hits / tasks.length, + held_out_task_pass_count: null, + context_tokens: contextTokens, + query_latency_ms: { + mean: latencies.reduce((a, b) => a + b, 0) / latencies.length, + p95: percentile(latencies, 0.95), + }, + stale_context_rate: records ? stale / records : 0, + agent_calls: 0, + cost_usd: 0, + tasks: taskResults, + }; +} + +export function runGraphContextBenchmark({ projectRoot, datasetPath }) { + const dataset = JSON.parse(readUtf8RegularFile(datasetPath)); + if ( + dataset.split !== "held-out" || + dataset.tasks?.length < 50 || + dataset.task_count !== dataset.tasks.length + ) + throw new Error("graph context benchmark requires at least 50 frozen held-out tasks"); + const projectionStarted = performance.now(); + const manifest = projectGraph({ projectRoot }); + const projectionTime = performance.now() - projectionStarted; + const graph = loadGraph(projectRoot, manifest.run_id); + const files = graph.nodes + .filter((node) => node.kind === "File") + .map((node) => ({ + ...node, + snippet: readUtf8RegularFile( + projectSourcePath(projectRoot, node.attributes.path), + 1_048_576, + ).slice(0, 2000), + })); + const modes = [ + evaluateMode("current-context", dataset.tasks, (task) => flatRank(files, task.query, false)), + evaluateMode("lexical", dataset.tasks, (task) => flatRank(files, task.query, true)), + evaluateMode("lexical-plus-graph", dataset.tasks, (task) => + graphRank(projectRoot, manifest.run_id, task, false), + ), + evaluateMode("graph-plus-promoted-memory", dataset.tasks, (task) => + graphRank(projectRoot, manifest.run_id, task, true), + ), + ]; + const protectedLeakage = modes.some((mode) => + mode.tasks.some((task) => + task.selected_paths.some((path) => + /(^|\/)(?:\.env(?:\.|$)|\.git|\.ssh|\.aws|\.gnupg)(\/|$)|\.(?:pem|key|p12|pfx)$/i.test( + path, + ), + ), + ), + ); + return { + schema_version: "1.0.0", + dataset_id: dataset.dataset_id, + split: dataset.split, + task_count: dataset.tasks.length, + repository_id: manifest.repository_id, + snapshot_id: manifest.snapshot_id, + projection_time_ms: projectionTime, + projection_node_count: manifest.node_count, + projection_edge_count: manifest.edge_count, + cross_project_leakage: false, + protected_path_leakage: protectedLeakage, + modes, + experimental_exit_criteria_evaluated: true, + experimental_exit_criteria_passed: false, + interpretation: + "Task pass count requires provider execution and is not inferred from localization. No release-status transition is made by this retrieval-only run.", + }; +} + +function main() { + const options = parse(process.argv.slice(2)); + const projectRoot = resolve(options.projectRoot ?? process.cwd()); + const datasetPath = resolve( + options.dataset ?? + resolve(projectRoot, "evals/datasets/graph-context/graph-context-held-out.json"), + ); + const result = runGraphContextBenchmark({ projectRoot, datasetPath }); + const body = `${JSON.stringify(result, null, 2)}\n`; + if (options.output) writePrivateUtf8File(resolve(options.output), body); + process.stdout.write(body); +} + +if (process.argv[1] && resolve(process.argv[1]) === resolve(import.meta.filename)) { + try { + main(); + } catch (error) { + process.stderr.write(`ERROR: ${error.message}\n`); + process.exitCode = 1; + } +} diff --git a/packages/orchestration/scripts/eval/run-matrix.mjs b/packages/orchestration/scripts/eval/run-matrix.mjs index c9a6879..7aecb0b 100755 --- a/packages/orchestration/scripts/eval/run-matrix.mjs +++ b/packages/orchestration/scripts/eval/run-matrix.mjs @@ -2,12 +2,11 @@ /** * Executes evaluation task/configuration matrices through the pipeline while preserving run isolation. */ -import { existsSync, mkdirSync } from "node:fs"; import { resolve } from "node:path"; import { spawnSync } from "node:child_process"; import { validateTasksetSchema } from "./lib/taskset-validate.mjs"; import { - getRunDir, + readJson, readJsonStrict, resolveWithinRepo, toWorkspaceRelative, @@ -79,8 +78,8 @@ function parseRunId(initOutput, root) { throw new Error(`Could not parse run_id from pipeline-init output:\n${initOutput}`); } const runId = match[1]; - const runDir = getRunDir(runId, root); - if (!existsSync(runDir)) { + const state = readJsonStrict(resolve(root, ".pipeline", "pipeline-state.json")); + if (state.run_id !== runId) { throw new Error(`pipeline-init returned run_id without run directory: ${runId}`); } return runId; @@ -151,10 +150,10 @@ function validateTaskset(taskset) { function loadTaskset(root, tasksetRef) { const abs = resolveWithinRepo(tasksetRef, root); - if (!existsSync(abs)) { + const taskset = readJson(abs, null); + if (!taskset) { throw new Error(`Taskset not found: ${tasksetRef}`); } - const taskset = readJsonStrict(abs); validateTasksetSchema({ root, tasksetPath: tasksetRef, @@ -234,19 +233,15 @@ function executeRun({ root, runId, configId, tasksetRel, taskId }) { return { failed }; } -function main() { - const args = parseArgs(process.argv); - const root = resolve(args.root); - const { taskset, tasksetRel } = (() => { - const loaded = loadTaskset(root, args.taskset); - return { taskset: loaded.taskset, tasksetRel: loaded.rel }; - })(); - - const evalDir = resolveWithinRepo(`.pipeline/evaluations/${args.evalId}`, root); - const matrixPath = resolve(evalDir, "matrix.json"); - const reportPath = resolve(evalDir, "evaluation-report.json"); - mkdirSync(evalDir, { recursive: true }); +function evaluationPaths(root, evalId) { + const evalDir = resolveWithinRepo(`.pipeline/evaluations/${evalId}`, root); + return { + matrixPath: resolve(evalDir, "matrix.json"), + reportPath: resolve(evalDir, "evaluation-report.json"), + }; +} +function executeMatrix({ args, root, taskset, tasksetRel }) { const runIdsByConfig = new Map(CONFIG_IDS.map((id) => [id, []])); const runMeta = []; @@ -277,8 +272,11 @@ function main() { } } } + return { runIdsByConfig, runMeta }; +} - const matrix = { +function matrixArtifact({ args, taskset, runIdsByConfig, runMeta }) { + return { evaluation_id: args.evalId, taskset_id: taskset.taskset_id, mode: args.mode, @@ -289,9 +287,9 @@ function main() { })), run_meta: runMeta, }; +} - writeJson(matrixPath, matrix); - +function aggregateMatrix({ args, root }) { runCommand( "node", [ @@ -305,7 +303,9 @@ function main() { ], { cwd: root }, ); +} +function emitMatrixSummary({ matrixPath, reportPath, runMeta }) { const failedRuns = runMeta.filter((entry) => entry.failed); process.stdout.write( `${JSON.stringify( @@ -324,6 +324,17 @@ function main() { } } +function main() { + const args = parseArgs(process.argv); + const root = resolve(args.root); + const { taskset, rel: tasksetRel } = loadTaskset(root, args.taskset); + const { matrixPath, reportPath } = evaluationPaths(root, args.evalId); + const { runIdsByConfig, runMeta } = executeMatrix({ args, root, taskset, tasksetRel }); + writeJson(matrixPath, matrixArtifact({ args, taskset, runIdsByConfig, runMeta })); + aggregateMatrix({ args, root }); + emitMatrixSummary({ matrixPath, reportPath, runMeta }); +} + try { main(); } catch (error) { diff --git a/packages/orchestration/scripts/eval/workflow-topology-benchmark.mjs b/packages/orchestration/scripts/eval/workflow-topology-benchmark.mjs new file mode 100644 index 0000000..7b8c861 --- /dev/null +++ b/packages/orchestration/scripts/eval/workflow-topology-benchmark.mjs @@ -0,0 +1,89 @@ +#!/usr/bin/env node +/** Emits a deterministic fixture for workflow event order, critical path, and barrier idle time. */ +import { existsSync, lstatSync, mkdirSync, realpathSync, writeFileSync } from "node:fs"; +import { resolve } from "node:path"; + +const items = [ + { key: "a", first: 8, second: 2 }, + { key: "b", first: 3, second: 6 }, + { key: "c", first: 5, second: 1 }, +]; +const entryDuration = 4; +const firstCompletion = items.map((item) => ({ ...item, completed: entryDuration + item.first })); +const barrierOpen = Math.max(...firstCompletion.map(({ completed }) => completed)); +const barrierCompletion = barrierOpen + Math.max(...items.map(({ second }) => second)); +const streamCompletion = Math.max( + ...firstCompletion.map(({ completed, second }) => completed + second), +); +const barrierIdle = firstCompletion.reduce( + (total, { completed }) => total + barrierOpen - completed, + 0, +); +const eventOrder = [ + { event: "entry_completed", at_ms: entryDuration }, + ...firstCompletion + .map(({ key, completed }) => ({ + event: "first_stage_completed", + item_key: key, + at_ms: completed, + })) + .sort((left, right) => left.at_ms - right.at_ms || left.item_key.localeCompare(right.item_key)), + ...firstCompletion + .map(({ key, completed, second }) => ({ + event: "stream_stage_completed", + item_key: key, + at_ms: completed + second, + })) + .sort((left, right) => left.at_ms - right.at_ms || left.item_key.localeCompare(right.item_key)), +]; + +const result = { + schema_version: "1.0.0", + fixture_id: "workflow-topology-order-v1", + measurements: { + event_order: eventOrder, + streaming_critical_path_ms: streamCompletion, + barrier_critical_path_ms: barrierCompletion, + barrier_idle_time_ms: barrierIdle, + }, + interpretation: { + scope: "deterministic scheduler fixture", + model_quality_claim: false, + universal_speed_claim: false, + }, +}; + +const OUTPUT_PATH = "eval-results/workflow-topology.json"; + +function outputPath(pathValue, cwd = process.cwd()) { + if (!pathValue || pathValue.startsWith("--")) throw new Error("--output requires a path"); + if (pathValue !== OUTPUT_PATH) { + throw new Error(`--output must be ${OUTPUT_PATH}`); + } + mkdirSync("eval-results", { recursive: true, mode: 0o700 }); + if (realpathSync("eval-results") !== resolve(cwd, "eval-results")) { + throw new Error("eval-results/ must not be a symbolic link"); + } + if ( + existsSync("eval-results/workflow-topology.json") && + lstatSync("eval-results/workflow-topology.json").isSymbolicLink() + ) { + throw new Error("--output must not replace a symbolic link"); + } + return OUTPUT_PATH; +} + +function writeResult(pathValue) { + outputPath(pathValue); + writeFileSync("eval-results/workflow-topology.json", `${JSON.stringify(result, null, 2)}\n`, { + encoding: "utf8", + mode: 0o600, + }); +} + +const outputIndex = process.argv.indexOf("--output"); +if (outputIndex >= 0) { + writeResult(process.argv[outputIndex + 1]); +} else { + process.stdout.write(`${JSON.stringify(result, null, 2)}\n`); +} diff --git a/packages/orchestration/scripts/pipeline/autonomous.mjs b/packages/orchestration/scripts/pipeline/autonomous.mjs index ffc5f44..03f9583 100644 --- a/packages/orchestration/scripts/pipeline/autonomous.mjs +++ b/packages/orchestration/scripts/pipeline/autonomous.mjs @@ -34,11 +34,17 @@ Run options: --provider auto or codex (command is test-integration only) --model Optional Codex model override --reasoning-effort low, medium, high, or xhigh + --execution-profile Operator-owned logical tier to Codex mapping --timeout-seconds Per-phase timeout (default: 1800) --policy Validated data-only autonomous policy JSON + --workflow Explicit graph-native workflow JSON for a new run + --legacy-linear Start a temporary v1 ten-phase run --checkpoint-policy Human pause mode: none, before-mutation, or before-mutation-and-ship + --graph-memory Local graph mode: off, read, or read-write (default: off) --in-place Modify a clean target checkout directly - --through Stop after one named phase (default: release-readiness) + --through Stop after one workflow node + --max-concurrency Concurrent readers, from 1 to 4 (default: 4) + --max-repair-rounds Repair iterations, from 1 to 5 (default: 5) --run-id Resume an existing run (resume command only) --json Emit the final result as JSON @@ -57,8 +63,14 @@ Safety defaults: } function parseOptions(argv) { - const options = { _: [], agentArgs: [] }; - const booleanFlags = new Set(["in-place", "json", "help", "allow-unsafe-command-provider"]); + const options = Object.assign(Object.create(null), { _: [], agentArgs: [] }); + const booleanFlags = new Set([ + "in-place", + "json", + "help", + "legacy-linear", + "allow-unsafe-command-provider", + ]); for (let index = 0; index < argv.length; index++) { const token = argv[index]; if (!token.startsWith("--")) { @@ -67,7 +79,7 @@ function parseOptions(argv) { } const key = token.slice(2); if (booleanFlags.has(key)) { - options[key] = true; + Reflect.set(options, key, true); continue; } const value = argv[index + 1]; @@ -78,19 +90,19 @@ function parseOptions(argv) { if (key === "agent-arg") { options.agentArgs.push(value); } else { - options[key] = value; + Reflect.set(options, key, value); } } return options; } -function main() { +async function main() { const [command = "help", ...rest] = process.argv.slice(2); const options = parseOptions(rest); if (isHelpCommand(command, options)) return usage(); if (command === "doctor") return runDoctor(options); if (["status", "stop", "resolve-checkpoint", "events"].includes(command)) return runControlCommand(command, options); - if (["run", "resume"].includes(command)) return runWorkflow(command, options); + if (["run", "resume"].includes(command)) return await runWorkflow(command, options); throw new Error(`unknown autonomous command: ${command}`); } @@ -109,7 +121,7 @@ function runDoctor(options) { if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { try { - main(); + await main(); } catch (error) { process.stderr.write(`ERROR: ${error.message}\n`); process.exitCode = 1; diff --git a/packages/orchestration/scripts/pipeline/graph-cli.mjs b/packages/orchestration/scripts/pipeline/graph-cli.mjs new file mode 100644 index 0000000..bc5e08d --- /dev/null +++ b/packages/orchestration/scripts/pipeline/graph-cli.mjs @@ -0,0 +1,312 @@ +#!/usr/bin/env node +/** Exposes local graph projection, query, explanation, and memory lifecycle commands. */ +import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { assertSupportedNodeRuntime } from "../lib/node-runtime.mjs"; +import { + decideMemory, + explainGraphNode, + graphStatus, + listMemory, + memoryStatus, + projectGraph, + queryGraph, + rebuildMemory, + recordRunMemory, +} from "./lib/graph.mjs"; +import { loadWorkflow } from "./lib/workflow-contract.mjs"; +import { createWorkflowRegistry } from "./lib/workflow-registry.mjs"; +import { proposeWorkflow } from "./lib/workflow-proposal.mjs"; + +assertSupportedNodeRuntime(); + +function usage() { + process.stdout.write(`RAE local graph engineering and memory + +Usage: + ./scripts/rae.sh graph build --project-root [--run-id ] [--json] + ./scripts/rae.sh graph status --project-root [--run-id ] [--json] + ./scripts/rae.sh graph query --project-root --seed [--run-id ] [--phase ] [--depth <0..4>] [--limit <1..200>] [--include-model-proposed] [--json] + ./scripts/rae.sh graph explain --project-root --run-id --node [--json] + ./scripts/rae.sh graph memory list --project-root [--status all|facts|candidates] [--json] + ./scripts/rae.sh graph memory promote|reject --project-root --candidate-id --actor --rationale --source-ref [--json] + ./scripts/rae.sh graph memory rebuild --project-root [--run-id ] [--json] + ./scripts/rae.sh graph workflow list --project-root [--json] + ./scripts/rae.sh graph workflow show --project-root --workflow [--json] + ./scripts/rae.sh graph workflow validate --project-root (--workflow-file | --workflow --revision ) [--json] + ./scripts/rae.sh graph workflow diff --project-root --workflow --from --to [--json] + ./scripts/rae.sh graph workflow activate --project-root --workflow --revision --digest --actor