diff --git a/config/reliability-gates.jsonc b/config/reliability-gates.jsonc index ce712d1c01b9..a6a291c57e1a 100644 --- a/config/reliability-gates.jsonc +++ b/config/reliability-gates.jsonc @@ -12276,6 +12276,7 @@ "surfaces": [ "live-idle terminal notification", "Run binding and rebinding", + "Run coordinator process authority", "active Dispatch ownership", "orchestration check and acknowledgment", "outstanding Run Deliveries", @@ -12286,17 +12287,19 @@ "providers": ["local", "daemon", "ssh", "wsl", "remote-runtime"], "coveredPlatforms": ["macos"], "coveredProviders": ["local", "daemon", "ssh"], - "coverageNotes": "A deterministic isolated runtime/SQLite/RPC harness covers Run and Dispatch ownership, exact handle and remint-stable pane recipients, fixed Delivery replay and acknowledgment, restart, concurrent pollers, filtered waiters, and built-CLI parity with direct DB state. Schema v28 durably records every coordinator handle through SQLite triggers, backfills current handles on open, recreates routing triggers after old table rebuilds, and atomically canonicalizes coordinator-addressed inserts to run:. A 125-row oracle and 151-row reconciliation suite cover paged old-handle backlogs, commit-without-notify recovery, duplicate Dispatch owners, cancellation, and lifecycle fencing. Federation tests use real databases, dropped acknowledgments, restart/reconnect, and protocol v1/v2 peers; duplicate read rows do not wake again. Daemon and SSH pointer settlement is deterministic, with SSH bounded at 60 seconds beyond the 20-second mux health window. No RPC parameter, stream opcode, or published payload changed. Live headed/headless paired-runtime, physical SSH, WSL, Linux, and Windows acceptance remain gaps.", + "coverageNotes": "A deterministic isolated runtime/SQLite/RPC harness covers Run and Dispatch ownership, exact handle and remint-stable pane recipients, fixed Delivery replay and acknowledgment, restart, concurrent pollers, filtered waiters, and built-CLI parity with direct DB state. Schema v28 durably records every coordinator handle through SQLite triggers, backfills current handles on open, recreates routing triggers after old table rebuilds, and atomically canonicalizes coordinator-addressed inserts to run:. Schema v31 adds coordinator process incarnation, execution-host scope, and an authority revision; tests initialize newly created post-migration Runs at revision 0, refuse live and unverifiable replacement, reject identity-less copied-handle mutations and generic replies, require runtime-attested identity before explicit reads claim current-consumer authority, require runtime-attested restored-process continuity before backfilling a migrated revision -1 row, fence attestation loss during that bind, preserve generation and pending Delivery identity for the same process after handle/pane remint even before the agent hook observes its reminted pane, permit only owning-host-proven exit takeover, fence a stale concurrent takeover observation, and clear stale authority when an older runtime changes only v30 binding columns. Lifecycle tokens bypass caller attestation only for genuinely capability-backed Dispatch rows; legacy replies reject contradictory declared senders. A 125-row oracle and 151-row reconciliation suite cover paged old-handle backlogs, commit-without-notify recovery, duplicate Dispatch owners, cancellation, and lifecycle fencing. Federation tests use real databases, dropped acknowledgments, restart/reconnect, and protocol v1/v2 peers; duplicate read rows do not wake again. Daemon and SSH pointer settlement is deterministic, with SSH bounded at 60 seconds beyond the 20-second mux health window. RPC changes are additive optional response fields; no stream opcode changed. Live headed/headless paired-runtime, physical SSH, WSL, Linux, and Windows acceptance remain gaps.", "motivatingLinks": [ + "https://linear.app/stably/issue/STA-5729", "https://linear.app/stably/issue/STA-4325", "https://github.com/stablyai/orca/issues/14522", "internal incident evidence: improve-vps-setup, 2026-08-10" ], - "invariant": "Each message has one stable row ID and authoritative recipient; coordinator-addressed current-delivery inserts are atomically owned by run:. Pointer staging may set delivered_at but never consumes mail. Each Run consumer generation has at most one outstanding Delivery with a fixed ID and fixed message IDs; ordinary checks replay it until an explicit matching acknowledgment marks exactly those rows read. Rebinding fences the old generation, notification types/counts correspond to unread rows retrievable under the same authority, and federation replay imports each stable message identity once without re-waking an already-read duplicate.", - "oracle": "Seed status, dispatch, and worker_done rows across direct-handle and canonical Run recipients in an isolated DB. Compare pointer count, RPC and built-CLI check output, direct SQLite rows, unread/peek/all/type filters, concurrent pollers, fixed Delivery IDs, explicit acknowledgment, restart, filtered check --wait, and coordinator remint. Route a 125-row old-handle backlog, inject a commit without notification, and require startup repair. Exercise duplicate Run/Dispatch owners, stale panes, 50-row pages, cancellation, lifecycle fencing, and absent PTYs. Drop a federation ACK, reconnect/restart v1/v2 peers, and require stable import plus no duplicate read-row wake. Hold a healthy SSH write past five seconds but below the 60-second settlement deadline, then separately exceed the bound and require retryable undelivered state.", + "invariant": "Each message has one stable row ID and authoritative recipient; coordinator-addressed current-delivery inserts are atomically owned by run:. Pointer staging may set delivered_at but never consumes mail. Each Run consumer generation has at most one outstanding Delivery with a fixed ID and fixed message IDs; ordinary checks replay it until an explicit matching acknowledgment marks exactly those rows read. Coordinator authority follows the same process incarnation on the same execution host across handle/pane remint without fencing; a distinct process can replace it only after the owning host proves the incumbent exited, and a stale observation cannot win. Rebinding fences the old generation, notification types/counts correspond to unread rows retrievable under the same authority, and federation replay imports each stable message identity once without re-waking an already-read duplicate.", + "oracle": "Seed status, dispatch, and worker_done rows across direct-handle and canonical Run recipients in an isolated DB. Compare pointer count, RPC and built-CLI check output, direct SQLite rows, unread/peek/all/type filters, concurrent pollers, fixed Delivery IDs, explicit acknowledgment, restart, filtered check --wait, and coordinator remint. Attempt live, unverifiable, proven-exited, same-process, cross-host, and stale-observation Run bindings; require one authority winner, preserved pending Delivery identity for continuity, and explicit read-only disclosure for a non-owner listing. Route a 125-row old-handle backlog, inject a commit without notification, and require startup repair. Exercise duplicate Run/Dispatch owners, stale panes, 50-row pages, cancellation, lifecycle fencing, and absent PTYs. Drop a federation ACK, reconnect/restart v1/v2 peers, and require stable import plus no duplicate read-row wake. Hold a healthy SSH write past five seconds but below the 60-second settlement deadline, then separately exceed the bound and require retryable undelivered state.", "commands": [ "pnpm run build:cli && pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orchestration-message-delivery-identity.test.ts --reporter=dot --testTimeout=5000", "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orchestration-mailbox-routing-races.test.ts src/main/runtime/orchestration-mailbox-notification-consistency.test.ts src/main/runtime/orchestration-mailbox-detached-routing.test.ts src/main/runtime/orchestration-mailbox-transport-settlement.test.ts src/main/runtime/orchestration/run-coordinator-handle-migration.test.ts src/main/runtime/orchestration/orchestration-run-delivery-db.test.ts src/main/runtime/orchestration/orchestration-worker-dispatch-db.test.ts src/main/runtime/orchestration/formatter.test.ts src/main/providers/ssh-pty-provider.test.ts src/main/providers/ssh-pty-write.test.ts src/main/daemon/client.test.ts src/main/daemon/daemon-pty-router.test.ts src/main/daemon/degraded-daemon-pty-provider.test.ts", + "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orchestration/run-coordinator-authority-migration.test.ts src/main/runtime/orchestration/orchestration-run-delivery-db.test.ts src/main/runtime/orca-runtime-process-incarnation-liveness.test.ts src/main/runtime/rpc/methods/orchestration-runs.test.ts src/main/runtime/rpc/methods/orchestration-run-remint-authority.test.ts src/main/runtime/rpc/methods/orchestration-caller-attestation.test.ts src/main/runtime/rpc/orchestration-legacy-compatibility-dispatcher.test.ts src/main/runtime/rpc/methods/orchestration-ask.test.ts src/cli/handlers/orchestration-run-cli.test.ts src/main/ipc/pty-controller-process-inventory.test.ts src/cli/handlers/orchestration-gate-cli.test.ts src/main/runtime/rpc/methods/orchestration-gates.test.ts src/main/runtime/rpc/methods/orchestration-gate-run-authorization.test.ts", "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orca-runtime.test.ts src/main/runtime/terminal-send-stale-leaf-liveness.test.ts src/main/runtime/rpc/methods/orchestration-runs.test.ts src/main/runtime/rpc/methods/orchestration-send.test.ts src/main/runtime/rpc/methods/orchestration-check.test.ts", "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orchestration/federation-sync.test.ts src/main/runtime/rpc/methods/orchestration-federation.test.ts src/main/runtime/rpc/methods/orchestration-federation-lifecycle-settlement.test.ts --reporter=dot" ], @@ -12307,7 +12310,14 @@ "src/main/runtime/orchestration-mailbox-routing-races.test.ts", "src/main/runtime/orchestration-mailbox-transport-settlement.test.ts", "src/main/runtime/orchestration/run-coordinator-handle-migration.test.ts", + "src/main/runtime/orchestration/run-coordinator-authority-migration.test.ts", + "src/main/runtime/orca-runtime-process-incarnation-liveness.test.ts", "src/main/runtime/orchestration/orchestration-run-delivery-db.test.ts", + "src/cli/handlers/orchestration-run-cli.test.ts", + "src/main/ipc/pty-controller-process-inventory.test.ts", + "src/cli/handlers/orchestration-gate-cli.test.ts", + "src/main/runtime/rpc/methods/orchestration-gates.test.ts", + "src/main/runtime/rpc/methods/orchestration-gate-run-authorization.test.ts", "src/main/runtime/orchestration/orchestration-worker-dispatch-db.test.ts", "src/main/runtime/orchestration/formatter.test.ts", "src/main/providers/ssh-pty-provider.test.ts", @@ -12317,6 +12327,10 @@ "src/main/daemon/degraded-daemon-pty-provider.test.ts", "src/main/runtime/orca-runtime.test.ts", "src/main/runtime/terminal-send-stale-leaf-liveness.test.ts", + "src/main/runtime/rpc/methods/orchestration-caller-attestation.test.ts", + "src/main/runtime/rpc/methods/orchestration-run-remint-authority.test.ts", + "src/main/runtime/rpc/orchestration-legacy-compatibility-dispatcher.test.ts", + "src/main/runtime/rpc/methods/orchestration-ask.test.ts", "src/main/runtime/rpc/methods/orchestration-runs.test.ts", "src/main/runtime/rpc/methods/orchestration-send.test.ts", "src/main/runtime/rpc/methods/orchestration-check.test.ts", @@ -12325,6 +12339,89 @@ "src/main/runtime/rpc/methods/orchestration-federation-lifecycle-settlement.test.ts" ], "assertionRefs": [ + { + "file": "src/main/runtime/orchestration/orchestration-run-delivery-db.test.ts", + "assertions": [ + "same-process handle remint preserves consumer generation and the outstanding Delivery ID", + "proven incumbent exit fences the old generation and replays unread mail to the winner", + "two takeovers using one exited-owner observation produce exactly one winner" + ] + }, + { + "file": "src/main/runtime/rpc/methods/orchestration-runs.test.ts", + "assertions": [ + "a live incumbent rejects a distinct coordinator with no effects", + "refuses replacement when a migrated incumbent is not provably exited", + "does not grant a migrated binding to a replacement incarnation on the same handle", + "backfills an exact migrated handle only from restored process authority and fences mid-bind attestation loss", + "backfills a migrated binding when a reminted handle resolves to the same process", + "same-process handle and pane remint preserves Run authority", + "does not grant Run authority to a replacement process in the same pane", + "a distinct coordinator binds only after owning-host process inventory proves exit", + "binds an authority-less migrated Run without requiring a nonexistent incumbent", + "cancels the Run bound during an asynchronous incumbent observation", + "explicit task listing discloses when the caller is not the current consumer" + ] + }, + { + "file": "src/main/runtime/rpc/methods/orchestration-run-remint-authority.test.ts", + "assertions": [ + "same-process remint survives stale agent-hook pane evidence only when current runtime launch proof matches the persisted process and host" + ] + }, + { + "file": "src/main/runtime/rpc/methods/orchestration-caller-attestation.test.ts", + "assertions": [ + "rejects an identity-less caller that copies the current coordinator handle with no effects and agent-readable recovery steps", + "marks identity-less explicit run-show, task-list, and gate-list reads that copy the current coordinator handle as non-owner inspection", + "a bogus lifecycle capability cannot bypass caller attestation for a capability-less Dispatch", + "an attested coordinator cannot reply to another Run mailbox" + ] + }, + { + "file": "src/main/runtime/rpc/orchestration-legacy-compatibility-dispatcher.test.ts", + "assertions": [ + "legacy direct reply rejects a declared sender that contradicts the attested coordinator" + ] + }, + { + "file": "src/main/runtime/orchestration/run-coordinator-authority-migration.test.ts", + "assertions": [ + "adds nullable authority identity to v30 Runs and backfills on same-pane use", + "creates new Runs at authority revision 0 after v31 migration", + "clears stale authority when an older runtime changes only the binding columns" + ] + }, + { + "file": "src/main/runtime/orca-runtime-process-incarnation-liveness.test.ts", + "assertions": [ + "keeps a live legacy-provider process without incarnation identity unverifiable" + ] + }, + { + "file": "src/cli/handlers/orchestration-run-cli.test.ts", + "assertions": [ + "keeps explicit Run inspection available without an active terminal", + "discloses a non-owner run-show as read-only", + "discloses headless run-show inspection as read-only", + "discloses non-owner inspection even when the Run has no tasks" + ] + }, + { + "file": "src/main/ipc/pty-controller-process-inventory.test.ts", + "assertions": [ + "bounds every relay list by the caller deadline instead of the mux default", + "forwards the caller deadline on a targeted local-provider list" + ] + }, + { + "file": "src/cli/handlers/orchestration-gate-cli.test.ts", + "assertions": ["inspects a named Run without requiring a caller terminal"] + }, + { + "file": "src/main/runtime/rpc/methods/orchestration-gate-run-authorization.test.ts", + "assertions": ["keeps gate-list from leaking gates outside the caller Run"] + }, { "file": "src/main/runtime/orchestration-message-delivery-identity.test.ts", "assertions": [ @@ -12433,6 +12530,15 @@ } ], "evidenceRuns": [ + { + "date": "2026-08-27", + "runner": "local", + "platform": "macos", + "command": "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orchestration/run-coordinator-authority-migration.test.ts src/main/runtime/orchestration/orchestration-run-delivery-db.test.ts src/main/runtime/orca-runtime-process-incarnation-liveness.test.ts src/main/runtime/rpc/methods/orchestration-runs.test.ts src/main/runtime/rpc/methods/orchestration-run-remint-authority.test.ts src/main/runtime/rpc/methods/orchestration-caller-attestation.test.ts src/main/runtime/rpc/orchestration-legacy-compatibility-dispatcher.test.ts src/main/runtime/rpc/methods/orchestration-ask.test.ts src/cli/handlers/orchestration-run-cli.test.ts src/main/ipc/pty-controller-process-inventory.test.ts src/cli/handlers/orchestration-gate-cli.test.ts src/main/runtime/rpc/methods/orchestration-gates.test.ts src/main/runtime/rpc/methods/orchestration-gate-run-authorization.test.ts", + "result": "passed", + "durationSeconds": 5.23, + "summary": "The final focused Run-authority gate passed 143/143 across schema migration and mixed-version writes, mutation and explicit-read caller attestation, same-process remint launch proof, capability-backed and capability-less Dispatch lifecycle authority, generic and legacy reply ownership, race and Delivery fencing, owning-host liveness and deadlines, coordinator lifecycle, and CLI recovery inspection. Sixteen authority regressions failed deterministically before their fixes; the final orchestration/RPC/CLI slice passed 1,298 tests across 135 files, after an earlier broader branch gate passed 2,405 tests with one skipped." + }, { "date": "2026-08-13", "runner": "local", @@ -12480,11 +12586,11 @@ }, "redGreenEvidence": { "status": "complete", - "evidence": "The byte-identical five-test oracle (SHA-256 76a2d672d27e04e5b1db2f52df3b303468bdf006be7cfd003bf443618a856c35) failed 5/5 on reported v1.4.180 commit 0b62333cf179, failed 5/5 on latest origin/main 9bb8836bb6, passed 5/5 on the final candidate, and failed 5/5 with candidate production changes reverted to 9bb8836bb6. Baselines announced two messages while check exposed one, returned no direct-only row after restart, left all 125 old-handle rows direct, failed commit-without-notify canonicalization, and made built CLI report one of two rows. The candidate aligned every count, stable message ID, fixed Delivery, acknowledgment, restart, backlog, and CLI/SQLite assertion. Live topology coverage remains a separate promotion gap." + "evidence": "The byte-identical five-test STA-4325 oracle (SHA-256 76a2d672d27e04e5b1db2f52df3b303468bdf006be7cfd003bf443618a856c35) failed 5/5 on reported v1.4.180 commit 0b62333cf179 and latest origin/main 9bb8836bb6, then passed 5/5 on its final candidate. For STA-5729, sixteen added regressions failed before implementation: same-pane process replacement mutated, an incarnation-less live coordinator was reported exited, an unbound migrated Run was stranded, a concurrent binding waiter was not cancelled, explicit headless Run inspection failed, targeted-local plus aggregate-local provider inventory dropped caller deadlines, a migrated exact-handle row accepted replacement or stale restored authority, an identity-less caller could copy the live coordinator handle for mutation or authority-bearing reads, post-migration Runs inherited revision -1, stale hook evidence rejected a proven same-process remint, bogus tokens bypassed caller attestation for capability-less Dispatches, generic replies crossed Run mailboxes, and legacy replies ignored contradictory declared senders. All sixteen passed after the authority, liveness, waiter, CLI, deadline, provenance, capability, reply-ownership, attestation, and revalidation fixes. The final orchestration/RPC/CLI slice passed 1,298 tests across 135 files after an earlier broader branch gate passed 2,405 tests with one skip. Live topology coverage remains a separate promotion gap." }, "performanceBudget": { "required": true, - "evidence": "Coordinator inserts use one SQLite trigger in the insertion transaction; current handles are backfilled and historical A-to-B-to-C bindings are trigger-recorded. Bulk rebind is one indexed O(backlog) UPDATE. Detached routing is an ownership-driven UNION whose coordinator, exact-Dispatch, and pane-Dispatch branches each cap at 51 duplicate-free rows before the final merge; full drains measured 40/77/187/1,040 ms for 5k/10k/20k/100k rows versus the prior 96/429/1,908 ms at 5k/10k/20k. Each transaction routes at most 50 rows and yields between pages. Typed partial indexes keep filtered Run peek at 0.08-0.13 ms over 100k rows. Trigger overhead measured about 0.6-1.0 microseconds per message. SSH pointer writes use one cleared, unrefed 60-second timer per pending write, beyond the 20-second mux health window. No subprocess, provider fanout, polling, persistent listener, renderer work, or wire payload was added." + "evidence": "Coordinator inserts use one SQLite trigger in the insertion transaction; current handles are backfilled and historical A-to-B-to-C bindings are trigger-recorded. Bulk rebind is one indexed O(backlog) UPDATE. Detached routing is an ownership-driven UNION whose coordinator, exact-Dispatch, and pane-Dispatch branches each cap at 51 duplicate-free rows before the final merge; full drains measured 40/77/187/1,040 ms for 5k/10k/20k/100k rows versus the prior 96/429/1,908 ms at 5k/10k/20k. Each transaction routes at most 50 rows and yields between pages. Typed partial indexes keep filtered Run peek at 0.08-0.13 ms over 100k rows. Trigger overhead measured about 0.6-1.0 microseconds per message. Distinct run-use performs one bounded owning-provider inventory only when authority continuity fails, passes the same three-second deadline to SSH/daemon transport, and retains an outer timeout. No polling, persistent listener, renderer work, stream opcode, or recurring subprocess was added." }, "promotionCriteria": [ "Add isolated live restart and SSH or paired-runtime evidence.", diff --git a/config/scripts/orchestration-skill-guidance.test.mjs b/config/scripts/orchestration-skill-guidance.test.mjs index 4eaf7d5753eb..f5095f73c5e2 100644 --- a/config/scripts/orchestration-skill-guidance.test.mjs +++ b/config/scripts/orchestration-skill-guidance.test.mjs @@ -51,6 +51,40 @@ describe('orchestration skill guidance', () => { ) }) + it('defines coordinator authority and ordinary takeover for agents', () => { + const skill = readSkill() + const authority = getSection(skill, 'Run Coordinator Authority') + + expect(authority).toContain('sole task-graph writer and consuming mailbox reader') + expect(authority).toContain('per-Run authority') + expect(authority).toContain('not an operating-system or global Orca privilege') + expect(authority).toContain('`run-use` is an authority claim, not a read-only selection') + expect(authority).toContain('`coordinatorStatus` is `live` or `unverifiable`') + expect(authority).toContain('`claimantStatus` is `changed`') + expect(authority).toContain('rejects the claim with `effectsApplied: false`') + expect(authority).toContain('a proven `exited` incumbent permits the claim to succeed') + expect(authority).toContain('`live`') + expect(authority).toContain('`unverifiable`') + expect(authority).toContain('`exited`') + expect(authority).toContain('Loss of contact is never evidence of exit') + expect(authority).toContain('No live-to-live transfer command exists') + expect(authority).toContain('`--takeover-legacy` is not a force override for ordinary Runs') + expect(authority).toContain('task-create') + expect(authority).toContain('task-update') + expect(authority).toContain('worker-start') + expect(authority).toContain('gate-create') + expect(authority).toContain('reply') + expect(authority).toContain('check') + expect(authority).toContain('run-show --id --json') + expect(authority).toContain('binding.currentConsumer') + expect(authority).toContain('never proves that the Run is unowned') + expect(authority).toContain('stop or exit the owning coordinator process') + expect(skill).not.toContain('unless impersonating another terminal') + expect(skill).toContain( + 'A declared handle is routing input, never proof of coordinator authority' + ) + }) + it('teaches attested adoption without reviving the retired scheduler', () => { const skill = readSkill() const migration = getSection(skill, 'Contract Migration') diff --git a/skill-guides/orchestration.md b/skill-guides/orchestration.md index 5236a7f47fc5..b21abe363f9f 100644 --- a/skill-guides/orchestration.md +++ b/skill-guides/orchestration.md @@ -99,6 +99,32 @@ Takeover fences only the old coordinator, binds the current one, and moves pendi Do not launch a replacement editor merely because the desktop app or runtime was updated. If adoption cannot prove continuing authority, keep the original worker as the only editor until it reaches a stable handoff point, then use a new current Dispatch in a conflict-free placement for any remaining work. +## Run Coordinator Authority + +Orca orchestration is agent-operated. A Run therefore has one current coordinator agent, not a human operator. That coordinator is the sole task-graph writer and consuming mailbox reader for coordinator-side operations; workers retain only their exact Dispatch capabilities for heartbeat, questions, escalation, and `worker_done`. This is per-Run authority, not an operating-system or global Orca privilege. + +The single-writer rule prevents two agents from changing Task, Dispatch, worker, or gate state concurrently and prevents two consumers from acknowledging different views of the same FIFO Delivery. The current coordinator may create and update Tasks (`task-create`, `task-update`), start or dispatch workers (`worker-start`, `dispatch`), create and resolve gates (`gate-create`, `gate-resolve`), answer worker questions (`reply`), and consume or acknowledge Run mail (`check`). Explicit `run-show`, `task-list --run`, `gate-list --run`, and inbox calls remain read-only for non-owners and headless callers. `check --peek` is non-consuming, but an ordinary current Run still requires its coordinator authority; do not use it as a non-owner inspection path. + +`run-use` is an authority claim, not a read-only selection. When a transfer is intended, make one claim from the stable replacement agent terminal with `orca orchestration run-use --id --json`; do not guess whether its process is a remint or spoof the owner with `--from`. Orca either preserves same-process authority, grants replacement authority after proving exit, or rejects the claim with `effectsApplied: false`. + +On rejection, read `error.data.coordinatorStatus`, `claimantStatus`, `nextSteps`, and any exact command-argument fields. `coordinatorStatus` is `live` or `unverifiable` because a proven `exited` incumbent permits the claim to succeed. If `claimantStatus` is `changed`, the invoking agent changed during proof; follow the returned retry arguments once from one stable process instead of treating it as a network verdict or retrying blindly. Never infer authority from a terminal handle alone. + +- Same coordinator process, reminted handle: authority and any outstanding Delivery are preserved without advancing the consumer generation. +- Different process and `live` incumbent: `consumer_fenced`; continue from the owning coordinator terminal. To transfer intentionally, stop or exit the owning coordinator process before retrying from its replacement. +- Different process and `unverifiable` incumbent: `consumer_fenced`; restore connectivity to the owning host. Loss of contact is never evidence of exit, including SSH, relay, Windows, WSL, and federated runtimes. +- Different process and `exited` incumbent: the replacement may run `orca orchestration run-use --id --json`. Orca advances the consumer generation, fences the old outstanding Delivery, preserves pending Run mail and worker assignments, and grants the replacement coordinator authority. + +Inspect without claiming authority: + +```bash +orca orchestration run-show --id --json +orca orchestration task-list --run --json +``` + +`binding.currentConsumer` is `true` only for the current coordinator. `false` permits inspection, not coordinator mutations or consuming `check` calls, and never proves that the Run is unowned. + +No live-to-live transfer command exists. A seamless live handoff would require a separate owner-authorized protocol; do not simulate one by retrying, changing `--from`, or replacing a terminal handle. `--takeover-legacy` is not a force override for ordinary Runs: it is limited to the automatically adopted legacy Run described above. + ## Ownership New orchestration messages and tasks belong to one explicitly bound Run. A Run is only a durable namespace and coordinator inbox; it never schedules or places workers. Lifecycle authority comes from the active Dispatch, and terminal handles remain routing metadata rather than durable identity. Send `worker_done` and `heartbeat` from the worker's own terminal; Orca routes them to that Dispatch's Run. @@ -135,7 +161,7 @@ orca orchestration inbox [--limit ] [--json] Rules: -- Omit `--from` unless impersonating another terminal; Orca auto-resolves it from the current terminal. +- Omit `--from` unless an injected preamble or exact recovery command supplies it; Orca normally resolves the current terminal. A declared handle is routing input, never proof of coordinator authority or a transfer mechanism. - A coordinator `check` returns the bound Run's oldest FIFO Delivery (up to 50 messages) and replays that exact batch until `--ack `. Process every message before acknowledging; `check --ack --wait` acknowledges, checks, and waits in one operation. - Use `--peek` and `--all` only for read-only history/debugging. Type filters decide when a waiter wakes; the returned actionable Delivery is still the oldest full batch. - Use `dispatch:` for coordinator guidance to one supervised worker. Orca routes that stable address locally or through the connected-server relay; do not substitute a remote terminal handle. @@ -198,7 +224,9 @@ Two limits worth knowing: - **It is a guardrail, not a security boundary.** A caller that declares another terminal's handle while its own launch evidence is unverifiable (an ordinary restored terminal, for - example) can be counted as that terminal instead. Orca does not treat workers as hostile. + example) can be counted as that terminal instead. This affects only nesting-depth + classification; it never grants coordinator or Dispatch authority. Orca does not treat + workers as hostile. - **It applies while a Dispatch is active.** After `worker_done`, or after a coordinator settles the task, the terminal is no longer a worker and is counted as a root again. The process may still be alive; that is the documented boundary, not an accident. diff --git a/src/cli/bundled-skill-guides.ts b/src/cli/bundled-skill-guides.ts index 327581b9d698..086247a22f45 100644 --- a/src/cli/bundled-skill-guides.ts +++ b/src/cli/bundled-skill-guides.ts @@ -30,7 +30,7 @@ const ORCA_LINEAR_MARKDOWN = "---\nname: orca-linear\ndescription: >-\n Use Orc const ORCA_PER_WORKSPACE_ENV_MARKDOWN = "---\nname: orca-per-workspace-env\ndescription: >-\n Set up, review, debug, or validate Orca per-workspace environment recipes —\n on-demand, disposable runtimes (cloud sandboxes, VMs, or local) created fresh\n for each workspace. Covers first-time setup (provider prerequisites, the\n reusable base snapshot, the coding-agent auth snapshot, credentials, and\n state), not just the per-workspace lifecycle scripts. Use to stand up\n per-workspace environments, fix an `environmentRecipes` entry in `orca.yaml`, scaffold\n provider lifecycle scripts, or resolve an `orca vm recipe doctor` failure.\n---\n\n# Per-Workspace Environments\n\nHelp a user stand up and maintain a repo-owned per-workspace environment recipe end to end. Each\nworkspace gets its own on-demand, disposable runtime (a cloud sandbox, a VM, or a local one),\ncreated fresh and torn down after.\n\nOrca is a **thin wrapper**: you guide, detect, and scaffold; you never own the user's cloud account,\nbilling, images, or credentials.\n\n- **You DO:** sequence the setup, detect what's detectable (provider CLI present/logged-in? recipe\n present? `doctor` passing?), scaffold provider-templated scripts the user fills in, drive the slow\n snapshot/auth phases with the user, and always show the next action.\n- **You DO NOT:** create accounts, choose plans/regions, invent org/project/scope ids, store or print\n secrets, or run anything that spends money without an explicit user OK.\n\nFirst-time setup has **four phases before the per-workspace recipe runs** — easy to miss, so walk\nthem in order:\n\n1. **Prerequisites** — cloud account, provider CLI, scope/project, plan limits, git token (§2).\n2. **Base snapshot** — reusable image: tools + repo + headless build, snapshotted once (§3).\n3. **Agent-auth snapshot** — boot the base, run interactive device-auth, re-snapshot (§4).\n4. **State** — thread snapshot id / scope / project / port between phases via a state file (§6).\n\nThen the **per-workspace contract** (create/suspend/resume/destroy) runs fast (§8).\n\n**The one branch that shapes everything — connection mode:** **Orca-server** (`create` runs `orca serve`\nin the env and emits a `pairingCode`; §7c/§7f) vs **SSH** (`create` runs no server and emits a\n`connection.type:\"ssh\"` block Orca dials into; §7g/§7h). Settle this first — it changes the `create`\noutput shape and half the templates.\n\nKeep Orca's checkout behavior unchanged by default: omit `checkoutMode`, emit schema version 1, and\nlet Orca create a linked worktree. Only use `checkoutMode: provisioned-root` when the user explicitly\nwants one ephemeral machine to clone the finished workspace itself. This niche mode currently requires\ndirect SSH, an ordinary non-bare/non-sparse primary checkout at `projectRoot`, and schema version 2.\n\n**Quick-start (happy path):** interview the user (connection mode Orca-server vs SSH, provider, agent CLI,\ngit auth — §1.2) + read the provider's CLI docs → scaffold `scripts/orca-vm/` from §7 → run the\nbase-snapshot script, then the auth script (you invoke these by hand; not via `orca.yaml`) → wire\n`environmentRecipes` in `orca.yaml` → `orca vm recipe doctor --json` (free) → then the `--provision`\nself-test loop (§9) until it passes.\n\n---\n\n## 1. Setup workflow\n\nDrive these with the user. **[CHECKPOINT]** steps need explicit confirmation — they spend money, take\na long time, or need the user at the keyboard. Never create an Orca workspace or commit unless asked.\n\n1. **Inspect the repo** for an existing `environmentRecipes` entry, `scripts/orca-vm/`, a state file, or setup\n notes. If a working recipe exists, jump to Doctor (§9) instead of rebuilding.\n2. **Interview the user up front** — gather these choices and confirm them back before scaffolding\n anything. Don't pick for them (§11); don't guess.\n - **Connection mode:** how Orca attaches to the environment — an **Orca server** (the VM runs\n `orca serve` and Orca pairs over its pairing URL; worked example §7f) or **SSH** (Orca connects to\n the host over SSH; §7g). This decides the recipe's connection shape, so settle it first.\n - **Checkout ownership:** do not ask by default. Only when the user requires the environment to\n create the exact final checkout, confirm `provisioned-root` and direct SSH; otherwise omit it.\n - **Provider:** Vercel Sandbox, Fly, Modal, an existing SSH host, … For non-obvious providers, also\n ask scope/project/region and plan limits (§2). Then **read that provider's CLI/SDK docs** (or\n ` --help`) before scaffolding — you need its exact create/exec/snapshot/remove verbs.\n If a provider advertises `ssh`, verify whether it exposes a real dialable SSH target\n (host/port/user/key or proxy command) or only a provider-mediated interactive shell; Orca SSH mode\n needs the former.\n - **Coding-agent CLI + account:** which agent runs in the VM (`codex`, `claude`, …) and that the user\n has an account for it — it gets logged in during the Phase-3 auth snapshot (§4).\n - **Git auth:** the token source for cloning a private repo (`GH_TOKEN`/`GITHUB_TOKEN` or `gh auth\ntoken`; §5).\n3. **Check prerequisites (§2)** — detect the provider CLI + auth and confirm the items above are in\n place before any paid step.\n4. **Scaffold scripts + state file** from §7 (worked Vercel example: §7f; SSH host: §7g; Docker SSH:\n §7h; Windows: §7i), filling in the provider's real commands. Make them executable.\n5. **[CHECKPOINT] Build the base snapshot (§3)** — paid, slow.\n6. **[CHECKPOINT] Authenticate the agent (§4)** — interactive; the user follows a URL/code. **You cannot\n drive this step** — you run commands non-interactively, so there's no TTY for `docker exec -it` /\n `ssh -t` to prompt against. The **user** runs the Phase-3 login in their own terminal (or via the\n Claude Code harness bang-prefix — `! `, with the required space after `!`); you scaffold and drive\n the non-interactive phases around it. After kicking it off, **ask the user to report back once the login\n finishes** — you can't observe it completing, and you need that confirmation before resuming the\n non-interactive steps (base/auth commit, doctor, provision).\n7. **Wire the recipe** so `orca.yaml` points create/suspend/resume/destroy at the scripts (§8). The\n workspace composer reads `environmentRecipes` from the project's primary checkout of `orca.yaml`, **not** from\n a feature branch or worktree. So a recipe added only on a branch won't appear as a \"Run on\" option\n until that `orca.yaml` change is committed and merged to the project's primary branch. Tell the user\n this up front: `doctor`/`--provision` validate the scripts from the working copy on any branch, but\n creating a workspace from the recipe in the picker needs it on primary.\n8. **Dry-run doctor** — `orca vm recipe doctor --repo-path --json` (free, static; §9).\n Fix every failure before going live.\n9. **[CHECKPOINT] Live self-test** — get the user's OK once, then run\n `orca vm recipe doctor --provision --json` as a loop: it runs create → validates →\n destroys, and on failure returns a full transcript. Read it, fix the scripts, and re-run yourself until\n it passes (§9). Spends cloud money; the one approval covers the loop.\n10. **[CHECKPOINT] Optional workspace test** — only if asked: create a workspace via the picker, then\n verify sleep/wake/delete.\n\n---\n\n## 2. Phase 1 — Prerequisites\n\nThe user's responsibility; verify what's verifiable, ask for the rest, invent nothing. State which\nitems you verified vs. which the user asserted.\n\n- **Connection mode** (Orca server vs SSH) confirmed with the user — see §1 step 2; it shapes the recipe.\n- **Cloud account + plan** that allows sandboxes/VMs. Ask.\n- **Provider CLI installed + authenticated** — detect (`command -v `), check auth (e.g.\n `vercel whoami`). If missing, point at the provider's docs; don't log them in.\n- **Scope / project / region** the sandboxes live under. Ask; flows into every script via state.\n- **Plan / timeout / RAM caps.** Record them — e.g. Vercel Hobby caps sandbox timeout at **45m**,\n which limits both the base build and per-workspace runtime (see §10).\n- **Git token for private repos** (`GH_TOKEN`/`GITHUB_TOKEN`, or the provider's git auth; can fall back\n to `gh auth token`). See §5.\n- **Coding-agent CLI choice** (`codex`, `claude`…) and that the user has an account — it gets\n authenticated into the VM in Phase 3.\n\n---\n\n## 3. Phase 2 — Base snapshot (the reusable image)\n\nBuild **once**, snapshot, and every workspace boots from it in seconds instead of rebuilding.\nProvisioning + building takes a while (often ~20–30 min), so it runs behind a checkpoint. The script\nshape is §7a; key points:\n\n- Build the **headless Electron main only** (not the renderer) so it fits in plan RAM.\n- Use the VM image's package manager (`apt`/`dnf`/`apk`, per the base distro — not the provider brand).\n- Clone with the git token via `GIT_ASKPASS` (§5).\n- **Trap errors and remove the half-built sandbox** so a crash doesn't leave a paid resource running.\n- **Never snapshot a machine on which the Orca runtime has already run.** The first `orca serve` creates\n the runtime's user-data dir, and everything in it gets baked into the image and shared by every VM\n booted from it: the pairing keypair and device-token registry (`orca-devices.json`,\n `orca-e2ee-keypair.json`), `agent-session-authority.key`, and the build box's logs, terminal history\n and orchestration db. Confirmed: two VMs from one such snapshot emitted **identical `deviceToken` and\n `pairedDeviceId`**. Snapshot **before** the runtime has ever run, or `rm -rf` the whole user-data dir\n (`~/.config/orca` on Linux) first — deleting a named file list will drift as Orca adds state.\n- Snapshot the stopped sandbox, parse the snapshot id, and write it + scope/project/port/repo to state.\n\n---\n\n## 4. Phase 3 — Agent-auth snapshot (interactive)\n\nThe base snapshot has the agent CLI installed but **not logged in**, and per-workspace VMs are\nephemeral — so authenticate once and bake it into a second snapshot layer. Script shape is §7b:\n\n1. Boot a sandbox from the base `snapshotId` (from state).\n2. Run the agent's login **interactively** (`--interactive --tty`); the user completes the URL/code in\n their browser. On a **headless VM this must be the device-auth flow** (e.g. `codex login --device-auth`),\n **not** plain `codex login`: the default OAuth login starts a loopback callback server on a container\n port the host browser can't reach, so it hangs. Device-auth instead prints a URL + code the user opens\n on the **host**.\n3. Verify login; **refuse to snapshot an unauthenticated VM.** Prefer the status command's **exit code**\n (most agent CLIs exit non-zero when unauthenticated). If you grep instead, agent status often goes to\n **stderr** (e.g. `codex login status` prints \"Logged in using ChatGPT\" there), so **fold stderr first**\n (`... 2>&1 | grep …`) and match the agent's **exact success line** — never `grep -qi 'logged in'`, which\n also matches \"**not** logged in\" and would commit an unauthenticated image.\n4. Re-snapshot, parse the new id, and overwrite `snapshotId` in state to the authenticated image\n (recording `authSourceSnapshotId`). Remove the auth sandbox.\n\n**You can't drive step 2 yourself** (you run commands non-interactively — no TTY). The **user** runs it in\ntheir own terminal, or via the Claude Code harness bang-prefix (`! `, with the required space after\n`!`). You scaffold/boot the sandbox and run steps 3–4, but **you cannot observe the interactive login\nfinishing** — so **ask the user to tell you when it's done** before you verify and re-snapshot.\n\nThis layer inherits §3's rule: if you started `orca serve` on the base or auth sandbox to smoke-test it,\ndelete the runtime's user-data dir (`~/.config/orca` on Linux) before re-snapshotting, or every workspace\nbooted from this image shares one pairing identity and one `agent-session-authority.key`.\n\nIf the agent's credentials are short-lived, warn that the snapshot may need periodic re-auth (§10).\n\nFor disposable runtimes, do **not** treat a host agent config directory (for example `~/.codex`) as the\nauth snapshot by bind-mounting or copying it wholesale. Agent homes often contain sqlite state, hook\napproval state, caches, logs, and host-specific env/config. Instead, authenticate/configure the agent\ninside the disposable runtime and snapshot/commit that runtime layer.\n\n---\n\n## 5. Credentials\n\n- **Never** commit secrets or put them in `userData`, recipe JSON, comments, docs, or the state file.\n- **Git token:** read from env (`GH_TOKEN`/`GITHUB_TOKEN`), falling back to `gh auth token`. Pass to the\n VM only via the provider's ephemeral `--env`. Inside the VM, use a `GIT_ASKPASS` helper with\n `x-access-token` (not the token in the clone URL) and `GIT_TERMINAL_PROMPT=0` so a missing token fails\n fast instead of hanging. When you write the helper from inside `bash -lc` under `set -u`, escape the\n positional arg and the token (`\\$1`, `\\$GH_TOKEN`) so they land **literally** and resolve at git-runtime\n — an unescaped `$1` aborts with \"unbound variable\", and a literal `$GH_TOKEN` keeps the real token out of\n the written file. `rm -f` the helper after the clone/fetch.\n- **Provider auth:** rely on the provider CLI's logged-in session, not checked-in keys.\n- **Agent auth:** lives in the authenticated snapshot (Phase 3) — never a file you write or commit.\n- State holds only **non-secret** wiring (snapshot ids, scope, project, port, repo url/ref).\n\n---\n\n## 6. State file\n\nA repo-local JSON file (e.g. `scripts/orca-vm/-state.json`) threads non-secret values between\nphases. Each script resolves values as **env var → state → built-in fallback**, and merges its outputs\nback. Phase 2 writes the base `snapshotId`; Phase 3 overwrites it with the authenticated snapshot;\nper-workspace `create` boots from `snapshotId`.\n\n```json\n{\n \"baseName\": \"orca-base\",\n \"snapshotId\": \"snap_authenticated_image_id\",\n \"authSourceSnapshotId\": \"snap_base_image_id\",\n \"scope\": \"\",\n \"project\": \"\",\n \"port\": 7331,\n \"repoUrl\": \"https://host/org/repo.git\",\n \"repoRef\": \"main\",\n \"projectRoot\": \"/abs/path/on/remote/repo\"\n}\n```\n\n---\n\n## 7. Script templates (provider-agnostic shapes)\n\nScaffold under `scripts/orca-vm/`. These are **shapes** — fill in the provider's real commands. All\nreserve stdout for the final JSON and log progress to stderr. Include a shared `json_value ` /\n`env_value ` reader (env → state → fallback) in each.\n\n**Where each script runs:**\n\n- **Local-side** (`create`/`suspend`/`resume`/`destroy` + the base-snapshot/auth scripts the user\n invokes) runs **on the user's desktop**, so it must run on their OS. macOS/Linux: `#!/usr/bin/env\nbash`, `set -euo pipefail`, quoted paths. **Windows:** a bare `.sh` won't run — scaffold `.ps1`/`.cmd`\n or require WSL/Git-Bash and point `orca.yaml` at the right launcher.\n- **Remote-side** (commands you `exec` _inside_ the Linux VM) always runs in the VM's Linux shell, so\n bash is fine there regardless of the user's OS.\n\n### 7a. Base-snapshot (`-base-snapshot.sh`) — Phase 2\n\n```bash\n#!/usr/bin/env bash\nset -euo pipefail\n# resolve base_name/repo_url/repo_ref/project_root/port/scope/project/timeout (env→state→fallback)\n# resolve gh token: GH_TOKEN | GITHUB_TOKEN | `gh auth token`\n# 1. provision a sandbox (timeout/vcpus/published port/snapshot retention); trap: remove on error\n# 2. remote exec (long timeout): install pkgs + gh + corepack/pnpm + agent CLI;\n# clone with GIT_ASKPASS(token); write headless main-only build config;\n# dev setup; pnpm install; build CLI; build headless electron main; smoke-check tools\n# 3. snapshot stopped sandbox; parse snapshot id (fail if unparseable)\n# 4. merge { baseName, snapshotId, projectRoot, repoUrl, repoRef, port, scope, project } into state\n# print only the state JSON to stdout\n```\n\nWorked Vercel commands for this phase are in §7f. You run this script by hand (not via `orca.yaml`),\nafter exporting the first-run inputs the state file doesn't have yet — e.g. provider scope/project, the\nrepo URL/ref, and a git token (`GH_TOKEN`); later runs read them back from state.\n\n### 7b. Auth (`-base-auth.sh`) — Phase 3\n\n```bash\n#!/usr/bin/env bash\nset -euo pipefail\n# read source snapshot from state.snapshotId (fail if absent); auth_name=\"${base_name}-auth\"\n# 1. boot sandbox from source snapshot; trap: remove on error\n# 2. INTERACTIVE/TTY remote exec: agent login — user completes URL/code. Headless VM: MUST use the\n# device-auth flow (e.g. `codex login --device-auth`) — plain OAuth login binds a loopback callback\n# port the host can't reach and hangs. User runs this themselves (you have no interactive TTY); ask\n# them to report back when it's done before continuing.\n# 3. verify login, then refuse to snapshot if not logged in. Prefer the status command's EXIT CODE (most\n# agent CLIs exit non-zero when unauthenticated) over string-matching. If you must grep, fold stderr\n# first (`status 2>&1 | grep …` — many agents print the success line there) and match the agent's exact\n# success line; never `grep -qi 'logged in'`, which also matches \"not logged in\". Codex example: §7f.\n# 4. snapshot; parse new id\n# 5. merge { snapshotId:, authSourceSnapshotId: } into state; remove auth sandbox\n# print only the state JSON to stdout\n```\n\n### 7c. Create (`-create.sh`) — per workspace\n\n```bash\n#!/usr/bin/env bash\nset -euo pipefail\n# read authenticated snapshotId/scope/project/port/repo*/project_root (env→state→fallback)\n# fail clearly if snapshotId is missing (point back to Phases 2–3)\n# name = orca-${ORCA_RECIPE_ID}-${ORCA_VM_INSTANCE_ID} (sanitized, length-capped)\n# 1. boot sandbox from snapshotId with a published port; capture the public URL → pairing address\n# (an externally reachable wss:// URL); trap: remove sandbox on error\n# 2. remote exec: ensure repo at desired commit; rebuild only if commit changed (cache marker)\n# 3. remote exec: start orca serve in the background and read the recipe JSON it writes (see below)\n# 4. print serve's JSON to stdout, optionally enriched with userData:\n# { schemaVersion:1, pairingCode, projectRoot, userData:{ provider, resourceId:name, snapshotId } }\n```\n\n**The exact `orca serve` invocation and its output (verified — do not improvise the flags).** Inside the\nVM, run:\n\n```bash\norca serve \\\n --port \"$PORT\" \\\n --project-root \"$ABS_REPO_PATH_ON_REMOTE\" \\\n --pairing-address \"$EXTERNAL_WSS_URL\" \\\n --recipe-json\n```\n\n**Binary name:** in a VM built from source (the Phase-2 flow), run it as `pnpm exec orca-dev serve …`\nfrom the repo root — `orca-dev` is the in-repo entrypoint and is what the §7f example uses. Plain\n`orca serve …` is the same command when the built CLI is installed on the VM's PATH. The flags/output\nare identical either way.\n\nThere is **no `--host` flag**. `--project-root` must be an absolute directory on the remote. With\n`--recipe-json` the server **stays running** and prints exactly this single object to **stdout**, then\nkeeps serving:\n\n```json\n{\n \"schemaVersion\": 1,\n \"pairingCode\": \"\",\n \"projectRoot\": \"\"\n}\n```\n\n`pairingCode` is the pairing URL, already pointing at whatever you passed as `--pairing-address` — so set\n`--pairing-address` to the externally reachable address and **pass `pairingCode` through unchanged; never\nhand-rewrite it**. Because serve runs in the foreground and doesn't exit, redirect its stdout to a file\nand poll until that file parses as JSON (and bail if the process dies — dump its stderr log). Your\n`create` script then prints that JSON (optionally merging `userData`). Concrete pattern: §7f.\n\n### 7d. Suspend / resume / destroy — per workspace\n\n```bash\n#!/usr/bin/env bash\nset -euo pipefail\npayload=\"$(cat)\" # Orca passes lifecycle JSON on stdin\nresource_id=\"$(node -e 'const d=JSON.parse(process.argv[1]); process.stdout.write(d.recipeResult?.userData?.resourceId ?? \"\")' \"$payload\")\"\n[ -n \"$resource_id\" ] || { echo \"No resource id in lifecycle payload\" >&2; exit 1; }\n# suspend: provider suspend \"$resource_id\"\n# resume: provider resume \"$resource_id\"; then RE-EMIT fresh recipe JSON (pairing may change)\n# destroy: provider remove \"$resource_id\" (or set destroy: none in orca.yaml)\n```\n\n### 7e. State file — scaffold with scope/project/repo filled in and snapshot ids empty (§6).\n\n### 7f. Worked example — Vercel Sandbox (all three phases)\n\nA real, working shape (the Vercel surface is a CLI: `vercel sandbox create|exec|snapshot|remove`). Adapt\nnames; verify flags against `vercel sandbox --help` for the user's CLI version before relying on them.\nThese ground §7a (base snapshot) and §7b (auth), which are otherwise generic skeletons.\n\n**Phase 2 — base snapshot (§7a):** provision → install tools + clone + headless build → snapshot.\n\n```bash\n# provision a fresh build sandbox (retain a couple of snapshots); trap-remove on error\nvercel sandbox create --name \"$base\" --runtime node24 --timeout 30m --vcpus 4 --publish-port \"$port\" \\\n --snapshot-expiration 30d --keep-last-snapshots 2 \"${vercel_args[@]}\" >&2\n# remote build (long timeout): install pkgs+gh+pnpm+agent CLI, clone with GIT_ASKPASS (write the helper\n# with LITERAL \\$1/\\$GH_TOKEN so they resolve at git-runtime, not write-time — see §5/§7f create — then\n# `rm -f /tmp/askpass.sh`), write the headless main-only build config (drop the renderer), dev setup,\n# build CLI + headless main, smoke-check\nvercel sandbox exec \"$base\" \"${vercel_args[@]}\" --timeout 25m --env \"GH_TOKEN=$gh_token\" … -- bash -lc '…build…' >&2\n# snapshot the STOPPED sandbox and parse the id from CLI output (fail if unparseable)\nout=\"$(vercel sandbox snapshot \"$base\" --stop --expiration 30d \"${vercel_args[@]}\" 2>&1)\"; printf '%s\\n' \"$out\" >&2\nsnapshot_id=\"$(printf '%s\\n' \"$out\" | sed -nE 's/.*(snap_[A-Za-z0-9]+).*/\\1/p' | tail -1)\"\n# merge { baseName, snapshotId, scope, project, port, repoUrl, repoRef, projectRoot } into state; print state JSON\n```\n\n**Phase 3 — agent-auth snapshot (§7b):** boot the base, log the agent in interactively, re-snapshot.\n(`codex` below is an example — substitute the user's chosen agent's login/status verbs, e.g. `claude`.)\n\n```bash\nvercel sandbox create --name \"$auth\" --snapshot \"$snapshot_id\" --timeout 30m --publish-port \"$port\" \"${vercel_args[@]}\" >&2\n# INTERACTIVE — the USER runs this in their own terminal (you have no interactive TTY) and completes the\n# URL/code on the HOST. --device-auth is MANDATORY on a headless VM: plain `codex login` binds a loopback\n# callback port the host browser can't reach and hangs. Ask the user to report back when login finishes.\nvercel sandbox exec --interactive --tty \"$auth\" \"${vercel_args[@]}\" -- bash -lc 'codex login --device-auth'\n# refuse to snapshot an unauthenticated VM — fold stderr, match codex's exact success line (§4)\nvercel sandbox exec \"$auth\" \"${vercel_args[@]}\" --timeout 30s -- bash -lc 'codex login status 2>&1' | grep -Eqi 'Logged in using ChatGPT|Logged in via device' \\\n || { echo \"agent not logged in; not snapshotting\" >&2; exit 1; }\nout=\"$(vercel sandbox snapshot \"$auth\" --stop --expiration 30d \"${vercel_args[@]}\" 2>&1)\"; printf '%s\\n' \"$out\" >&2\nnew_id=\"$(printf '%s\\n' \"$out\" | sed -nE 's/.*(snap_[A-Za-z0-9]+).*/\\1/p' | tail -1)\"\n# overwrite state.snapshotId = new_id, record authSourceSnapshotId = snapshot_id; remove the auth sandbox\n```\n\n**Per-workspace `create`** (the fast path):\n\n```bash\n#!/usr/bin/env bash\nset -euo pipefail\n# resolve from env→state→fallback: snapshot_id, scope, project, port, repo_url, repo_ref, project_root\nvercel_args=(); [ -n \"$scope\" ] && vercel_args+=(--scope \"$scope\"); [ -n \"$project\" ] && vercel_args+=(--project \"$project\")\n[ -n \"$snapshot_id\" ] || { echo \"snapshotId missing — run Phases 2–3 first\" >&2; exit 1; }\ngh_token=\"${GH_TOKEN:-${GITHUB_TOKEN:-$(command -v gh >/dev/null 2>&1 && gh auth token 2>/dev/null || true)}}\"\nrecipe_id=\"${ORCA_RECIPE_ID:-vercel-sandbox}\"\nrecipe_id=\"${recipe_id//./-}\" # Vercel names forbid dots.\ninstance_id=\"${ORCA_VM_INSTANCE_ID:-$(date +%s)}\"\nmax_recipe_id_length=$((128 - ${#instance_id} - 6)) # Preserve the unique instance suffix.\n[ \"$max_recipe_id_length\" -gt 0 ] || { echo \"ORCA_VM_INSTANCE_ID is too long for a Vercel sandbox name\" >&2; exit 1; }\nname=\"orca-${recipe_id:0:max_recipe_id_length}-${instance_id}\"\n\n# Arm cleanup BEFORE create so a failing create can't leak a half-built paid sandbox.\ncleanup_on_error() { [ \"$?\" -ne 0 ] && vercel sandbox remove \"$name\" \"${vercel_args[@]}\" >/dev/null 2>&1 || true; }\ntrap cleanup_on_error EXIT\n\n# 1. boot from the authenticated snapshot, publish the serve port\ncreate_output=\"$(vercel sandbox create --name \"$name\" --snapshot \"$snapshot_id\" \\\n --timeout 30m --publish-port \"$port\" \"${vercel_args[@]}\" 2>&1)\"; printf '%s\\n' \"$create_output\" >&2\n# Vercel prints the published https URL; derive the external wss:// pairing address from it\npublic_url=\"$(printf '%s\\n' \"$create_output\" | sed -nE 's#.*(https://[^[:space:]]+\\.vercel\\.run).*#\\1#p' | head -1)\"\n[ -n \"$public_url\" ] || { echo \"no published URL in create output\" >&2; exit 1; }\npairing_ws=\"${public_url/https:\\/\\//wss://}\"\n\n# 2. (remote) ensure the repo is at the right commit; rebuild only if the commit changed (cache marker)\nvercel sandbox exec \"$name\" \"${vercel_args[@]}\" --timeout 20m \\\n --env \"GH_TOKEN=$gh_token\" --env \"ORCA_PROJECT_ROOT=$project_root\" \\\n --env \"ORCA_REPO_URL=$repo_url\" --env \"ORCA_REPO_REF=$repo_ref\" \\\n -- bash -lc 'set -euo pipefail; cd \"$ORCA_PROJECT_ROOT\"; \\\n # Re-establish git auth for the private-repo fetch (why + full rationale: §5); else it hangs on a prompt.\n # Load-bearing escaping: \\$1 and \\$GH_TOKEN must land LITERALLY and resolve at git-runtime. Test after\n # any edit here — reformatting the nested printf/node quoting silently breaks the fetch or leaks the token.\n if [ -n \"${GH_TOKEN:-}\" ]; then \\\n printf \"%s\\n\" \"#!/usr/bin/env bash\" \"case \\\"\\$1\\\" in *Username*) echo x-access-token;; *Password*) echo \\\"\\$GH_TOKEN\\\";; esac\" > /tmp/askpass.sh; \\\n chmod 700 /tmp/askpass.sh; export GIT_ASKPASS=/tmp/askpass.sh GIT_TERMINAL_PROMPT=0; fi; \\\n git fetch origin \"$ORCA_REPO_REF\"; \\\n git checkout -B \"$ORCA_REPO_REF\" FETCH_HEAD; \\\n rm -f /tmp/askpass.sh; \\\n c=\"$(git rev-parse HEAD)\"; [ -f .orca-built ] && [ \"$(cat .orca-built)\" = \"$c\" ] || { \\\n pnpm install --prefer-offline && pnpm run build:cli && \\\n node config/scripts/run-electron-vite-build.mjs --config config/electron-vite.vm-serve.config.ts && \\\n printf \"%s\" \"$c\" > .orca-built; }' >&2\n\n# 3. (remote) start orca serve in the background, writing recipe JSON to a file; poll until it parses\nrecipe_json=\"$(vercel sandbox exec \"$name\" \"${vercel_args[@]}\" --timeout 60s \\\n --env \"ORCA_PORT=$port\" --env \"ORCA_PROJECT_ROOT=$project_root\" --env \"ORCA_PAIRING_ADDRESS=$pairing_ws\" \\\n -- bash -lc 'set -euo pipefail; cd \"$ORCA_PROJECT_ROOT\"; rm -f /tmp/orca-recipe.json /tmp/orca-serve.log; \\\n nohup pnpm exec orca-dev serve --port \"$ORCA_PORT\" --project-root \"$ORCA_PROJECT_ROOT\" \\\n --pairing-address \"$ORCA_PAIRING_ADDRESS\" --recipe-json >/tmp/orca-recipe.json 2>/tmp/orca-serve.log /dev/null 2>&1 && { cat /tmp/orca-recipe.json; exit 0; }; \\\n kill -0 \"$pid\" 2>/dev/null || { cat /tmp/orca-serve.log >&2; exit 1; }; sleep 0.25; \\\n done; cat /tmp/orca-serve.log >&2; echo \"serve recipe JSON timed out\" >&2; exit 1')\"\n\n# 4. print serve's JSON enriched with userData (single object on stdout)\nnode -e 'const p=JSON.parse(process.argv[1]); console.log(JSON.stringify({...p, schemaVersion:1,\n userData:{...p.userData, provider:\"vercel-sandbox\", resourceId:process.argv[2], snapshotId:process.argv[3]}}))' \\\n \"$recipe_json\" \"$name\" \"$snapshot_id\"\ntrap - EXIT\n```\n\n`suspend`/`resume`/`destroy` use `vercel sandbox stop|...|remove \"$resource_id\"` reading\n`userData.resourceId` from stdin (§7d). This is the **Orca-server** connection mode (the recipe emits a\npairing URL). If the user chose **SSH** in the §1 interview, use §7g instead.\n\n### 7g. Worked example — existing SSH host (SSH connection mode)\n\nSSH mode is **fundamentally different from §7c/§7f**, not a relabeling of them:\n\n- **`create` does NOT run `orca serve` and does NOT emit a `pairingCode`.** Orca itself connects to the\n host over its SSH relay, brings up the git + filesystem providers, and imports the repo. The script's\n only job is to make the host ready and **print SSH connection details** Orca will dial.\n- The result uses a `connection` block with `type: \"ssh\"` and a `target`, **not** the flat\n `pairingCode`/`projectRoot` shape. Exact shape (Orca rejects anything else):\n\n```json\n{\n \"schemaVersion\": 1,\n \"connection\": {\n \"type\": \"ssh\",\n \"projectRoot\": \"/abs/path/to/repo/on/host\",\n \"target\": {\n \"label\": \"my-box\",\n \"host\": \"192.0.2.10\",\n \"port\": 22,\n \"username\": \"ubuntu\",\n \"identityFile\": \"~/.ssh/id_ed25519\",\n \"jumpHost\": \"bastion.example.com\",\n \"proxyCommand\": \"cloudflared access ssh --hostname %h\",\n \"relayGracePeriodSeconds\": 0,\n \"portForwards\": []\n }\n }\n}\n```\n\n`label`, `host`, `port`, `username` are required; the rest are optional — omit any you don't need.\n\nFor an explicitly requested one-VM-per-workspace checkout, the create script must read\n`ORCA_RECIPE_RESULT_SCHEMA_VERSION`, `ORCA_REPO_URL`, `ORCA_REPO_REF`, `ORCA_REPO_REF_HEAD`, and\n`ORCA_REPO_BRANCH`. Use `ORCA_REPO_REF` to fetch the selected source, but create\n`ORCA_REPO_BRANCH` at the exact `ORCA_REPO_REF_HEAD` commit; resolving the symbolic ref again can race\nwith an upstream update. `ORCA_REPO_URL` and `ORCA_REPO_REF` are a matched fetch pair, including when\nthe desktop source uses multiple remotes. Return that primary checkout at `projectRoot` and emit the\nsame SSH result with:\n\n```bash\n[ -n \"${ORCA_REPO_REF_HEAD:-}\" ] || { echo \"missing pinned source commit\" >&2; exit 1; }\ngit fetch origin \"$ORCA_REPO_REF\"\ngit cat-file -e \"${ORCA_REPO_REF_HEAD}^{commit}\"\ngit checkout -B \"$ORCA_REPO_BRANCH\" \"$ORCA_REPO_REF_HEAD\"\n```\n\n```json\n{\n \"schemaVersion\": 2,\n \"checkoutMode\": \"provisioned-root\",\n \"connection\": {\n \"type\": \"ssh\",\n \"projectRoot\": \"/abs/repo\",\n \"target\": { \"label\": \"my-box\", \"host\": \"192.0.2.10\", \"port\": 22, \"username\": \"ubuntu\" }\n }\n}\n```\n\nFail if the requested schema is not `2`; do not silently fall back to the ordinary recipe shape.\n\n**Networking → which `target` fields to set** (how _your desktop_ reaches the box — there is no\n`orca serve` URL in SSH mode):\n\n- Public IP / DNS, or a Tailscale/VPN address → `host`; SSH port → `port` (usually 22).\n- Key auth → `identityFile` (add `identitiesOnly: true` if the agent has many keys).\n- Through a bastion → `jumpHost` (a `user@host` ProxyJump) **or** a full `proxyCommand` (e.g. an access\n proxy). Use one, not both.\n- A service port the workspace needs → add entries to `portForwards`.\n- `relayGracePeriodSeconds` (optional): how long Orca keeps the SSH relay alive after the workspace\n detaches before tearing it down; `0` = tear down immediately. Leave it off unless the user wants a\n reconnect grace window.\n\n**Toolchain & agent auth on a persistent (no-snapshot) host — do this ONCE, by hand, before wiring the\nrecipe** (there's no base image to bake; the host _is_ the base). Run the §7f Phase-2 install steps and\nthe §7f Phase-3 ` login --device-auth` **directly over SSH on the host** (interactive, e.g.\n`ssh -t user@host ' login --device-auth'`). After that the host stays ready across workspaces.\n\n```bash\n#!/usr/bin/env bash\nset -euo pipefail\n# resolve from env→state→fallback (default unset optionals to \"\"): ssh_username, host,\n# ssh_port (default 22), identity_file, jump_host, proxy_command, project_root, repo_url, repo_ref\n: \"${identity_file:=}\"; : \"${jump_host:=}\"; : \"${proxy_command:=}\" # avoid set -u aborts on optionals\ngh_token=\"${GH_TOKEN:-${GITHUB_TOKEN:-$(command -v gh >/dev/null 2>&1 && gh auth token 2>/dev/null || true)}}\"\nssh_target=\"${ssh_username}@${host}\"\nssh_opts=(-p \"$ssh_port\"); [ -n \"$identity_file\" ] && ssh_opts+=(-i \"$identity_file\")\n# Why: a fresh host's key isn't in known_hosts; a StrictHostKeyChecking prompt would HANG a\n# non-interactive create. Pre-add the key (or set the option) so it can't block.\nssh-keyscan -p \"$ssh_port\" \"$host\" >> \"$HOME/.ssh/known_hosts\" 2>/dev/null || true\n\n# 1. ensure the repo is present and at the right commit on the host (NO orca serve here)\nssh \"${ssh_opts[@]}\" \"$ssh_target\" \\\n \"GH_TOKEN='$gh_token' GIT_TERMINAL_PROMPT=0 bash -lc '\n set -euo pipefail\n [ -d \\\"$project_root/.git\\\" ] || git clone \\\"$repo_url\\\" \\\"$project_root\\\"\n cd \\\"$project_root\\\" && git fetch origin \\\"$repo_ref\\\" && git checkout -B \\\"$repo_ref\\\" FETCH_HEAD\n '\" >&2\n\n# 2. print the SSH connection block (NO pairingCode, NO orca serve). host/port/username tell Orca's\n# relay how to dial in; identityFile/jumpHost/proxyCommand/portForwards are emitted when set.\nnode -e 'const [host,port,user,idf,jh,pc,root]=process.argv.slice(1);\n const target={ label:\"per-workspace-host\", host, port:Number(port), username:user };\n if(idf) target.identityFile=idf; if(jh) target.jumpHost=jh; if(pc) target.proxyCommand=pc;\n // add target.portForwards=[...] here if the workspace needs forwarded service ports\n console.log(JSON.stringify({ schemaVersion:1, connection:{ type:\"ssh\", projectRoot:root, target } }))' \\\n \"$host\" \"$ssh_port\" \"$ssh_username\" \"$identity_file\" \"$jump_host\" \"$proxy_command\" \"$project_root\"\n```\n\n`suspend`/`resume`/`destroy`: on a persistent host there's usually nothing to tear down — set\n`destroy: none` and omit suspend/resume. (Orca still disconnects/reconnects its own SSH relay on\nsleep/wake/delete — that's separate from these scripts.)\n\nIf the SSH host is instead an **ephemeral/snapshot-capable VM** (your hypervisor, or a cloud VM with\nimage support), keep the §7f Phase-2/3 base-image model for provisioning, but still emit the\n`connection.type:\"ssh\"` block above instead of starting `orca serve`.\n\n### 7h. Worked example — local Docker SSH (SSH connection mode)\n\nLocal Docker can model an ephemeral SSH VM without cloud cost: build a base image with `sshd`, tools,\nrepo prerequisites, and the agent CLI; run an **interactive auth container** once; then `docker commit`\nthat container as the authenticated image used by per-workspace `create`.\n\nKey points:\n\n- Publish container SSH to a random localhost port (`-p 127.0.0.1::22`) and emit\n `connection.type:\"ssh\"` with `host:\"127.0.0.1\"`, that port, `username`, `identityFile`, and\n `identitiesOnly:true`.\n- Generate a repo-local SSH key if needed, but gitignore the private/public key files.\n- **Bake SSH host keys into the base image** (`ssh-keygen -A` at **build** time; at runtime only generate\n if absent). Ephemeral containers all present the **same** host key, so `known_hosts` on `127.0.0.1`\n doesn't churn as the published port rotates across workspaces (otherwise every container's freshly\n generated key collides on `localhost` and trips host-key-changed warnings).\n- The auth image is the Docker equivalent of Phase 3: the **user** runs the agent login **inside** the\n container (you can't drive it — you have no interactive TTY), configures proxy env/config, approves\n hooks, and you commit once they report it's done. On a headless container use the **device-auth** flow\n (§4). Verify login before committing — exit code, or fold stderr and match the exact success line (§4).\n- Do not bind-mount or copy the host's full agent home into the image. Let each container have writable\n agent state; only the committed auth image should carry reusable authenticated state.\n- If committing from an interactive shell, force the runtime entrypoint back to `sshd`:\n `docker commit --change='ENTRYPOINT [\"/usr/local/bin/orca-docker-ssh-entrypoint\"]' …`.\n- `destroy` should read `recipeResult.userData.resourceId` and run `docker rm -f \"$resource_id\"`.\n\nValidation before wiring/live use:\n\n```bash\ndocker image inspect \"$auth_image\" --format '{{json .Config.Entrypoint}}'\ndocker run -d --name \"$name\" -p 127.0.0.1::22 -e \"ORCA_SSH_PUBLIC_KEY=$pubkey\" \"$auth_image\"\ndocker ps -a --filter \"name=$name\"\ndocker logs \"$name\"\nssh -i \"$key\" -p \"$port\" -o IdentitiesOnly=yes user@127.0.0.1 'codex --version'\n```\n\nIf the container exits immediately, inspect logs before the cleanup trap removes it; a committed\ninteractive image with `ENTRYPOINT [\"bash\"]` is a common cause.\n\nAlso confirm the **host key is stable** across containers: the SSH `ssh -i … 127.0.0.1` dial should not\ntrigger a host-key-changed warning when a second container reuses the port. If it does, the host keys\nweren't baked into the base image (see the `ssh-keygen -A` point above).\n\n### 7i. Windows local-side scripts\n\nThe local-side scripts run on the user's desktop. On **Windows**, a bare `.sh` won't execute. Either\nrequire WSL/Git-Bash (and point `orca.yaml` at e.g. `bash ./scripts/orca-vm/.sh` via a `.cmd`\nlauncher), or scaffold PowerShell equivalents. Minimal PowerShell shape:\n\n```powershell\n#requires -Version 5\n$ErrorActionPreference = 'Stop'\n# resolve env→state→fallback; run the provider CLI / ssh the same way;\n# capture provider output; build the result object for the chosen mode and write ONE line of JSON to stdout.\n# Orca-server mode: @{ schemaVersion=1; pairingCode=$pairingCode; projectRoot=$projectRoot; userData=@{...} }\n# SSH mode: @{ schemaVersion=1; connection=@{ type=\"ssh\"; projectRoot=$projectRoot;\n# target=@{ label=$label; host=$host; port=$port; username=$user } } } (see §7g/§7h)\n($result | ConvertTo-Json -Compress -Depth 6)\n# progress/errors → Write-Error / the error stream, never stdout.\n```\n\nThe remote-side commands you run _inside_ the Linux VM stay bash regardless of the desktop OS.\n\n---\n\n## 8. Per-workspace recipe contract (the fast path)\n\nOnce the authenticated snapshot exists, this runs on every workspace create. Define recipes in\n`orca.yaml`:\n\n```yaml\nenvironmentRecipes:\n - id: cloud-sandbox\n name: Cloud Sandbox\n create: ./scripts/orca-vm/cloud-sandbox-create.sh\n suspend: ./scripts/orca-vm/cloud-sandbox-suspend.sh\n resume: ./scripts/orca-vm/cloud-sandbox-resume.sh\n destroy: ./scripts/orca-vm/cloud-sandbox-destroy.sh\n```\n\n`create` runs **locally from the repo root** and prints **one** JSON object to stdout. Its shape depends\non the connection mode chosen in §1:\n\n**Orca-server mode** — boot the env, start `orca serve` in it, and print serve's result:\n\n```json\n{\n \"schemaVersion\": 1,\n \"pairingCode\": \"orca-pairing-code-or-url\",\n \"projectRoot\": \"/absolute/path/to/repo/on/remote\",\n \"userData\": { \"provider\": \"example\", \"resourceId\": \"provider-resource-id\" }\n}\n```\n\nHere `pairingCode` (from `orca serve --recipe-json`) and `projectRoot` are required; `schemaVersion` (`1`)\nand `userData` are optional.\n\n**SSH mode** — do **not** run `orca serve`; print the `connection.type:\"ssh\"` block instead (full shape +\nworked script in §7g). `pairingCode` is **not** used in SSH mode.\n\n**Optional provisioned root** — only for direct SSH and only when explicitly requested. Add\n`checkoutMode: provisioned-root` to the recipe, require `ORCA_RECIPE_RESULT_SCHEMA_VERSION=2`, create\nthe requested `ORCA_REPO_BRANCH` at the pinned `ORCA_REPO_REF_HEAD` commit (use `ORCA_REPO_REF` only\nto fetch that commit) at the returned `projectRoot`, and emit schema version 2 with\n`checkoutMode: \"provisioned-root\"`. All recipes without this field retain the schema-v1 behavior above.\n\nLifecycle hooks (all run locally):\n\n- `create`: required. Prints recipe result JSON.\n- `suspend`: optional. Sleep; reads lifecycle payload on stdin.\n- `resume`: optional. Wake; reads payload on stdin and **prints fresh recipe JSON** (pairing may change).\n- `destroy`: optional unless `destroy: none`. Delete/cleanup; reads payload on stdin.\n\nStart Orca remotely with `orca serve --port \"$PORT\" --project-root \"$ABS_ROOT\" --pairing-address\n\"$EXTERNAL_WSS_URL\" --recipe-json` (exact flags + output in §7c). Set `--pairing-address` to the\nexternally reachable address so the emitted `pairingCode` is reachable; tunneling/port mapping is the\nscript's job.\n\nBackward compatibility: `command`→`create`, `cleanup`→`destroy`, `cleanup: none`→`destroy: none`.\nPrefer the lifecycle names.\n\n---\n\n## 9. Doctor and validation\n\nValidate in two stages — the cheap dry run first, then the live self-test.\n\n### Dry run (free, non-destructive) — always do this first\n\n`orca vm recipe doctor --repo-path --json` validates **static wiring only** — it does\n**not** boot anything. It checks: local-host execution (v1), repo path, recipe id exists,\ncreate/destroy/suspend/resume command paths resolve, suspend/resume are paired, and each script is\nexecutable (POSIX exec bit; skipped on Windows). Fix every failure here before spending any cloud money.\n\n### Live self-test (`--provision`) — diagnose and iterate yourself\n\n`orca vm recipe doctor --repo-path --provision --json` actually runs the recipe end\nto end: it executes `create`, validates the returned recipe JSON, then runs `destroy` to **tear the\nenvironment back down** (so the test leaves nothing running, as long as `destroy` works). It spends real\ncloud money, so get the user's OK **once** before starting — that one approval covers the whole loop\nbelow; do not re-ask before each run.\n\nOn failure, the JSON result includes a `provisionTranscript` with the **complete** captured output of\neach stage so you can self-diagnose without asking the user to relay logs:\n\n```json\n{\n \"ok\": false,\n \"checks\": [{ \"id\": \"recipe.provision\", \"status\": \"fail\", \"message\": \"…\" }],\n \"provisionTranscript\": {\n \"provision\": { \"exitCode\": 0, \"signal\": null, \"stdout\": \"…\", \"stderr\": \"…\", \"parseError\": \"…\" },\n \"destroy\": { \"exitCode\": 0, \"signal\": null, \"stdout\": \"…\", \"stderr\": \"…\" }\n }\n}\n```\n\n**Run it as a loop:** read `provisionTranscript.provision.stderr` / `.stdout` / `.parseError` (and\n`destroy.*`), fix the script, and re-run `--provision` until `ok` is `true` — iterating on your own\nrather than waiting for the user to paste errors. Common reads: a non-empty `stderr` with `exitCode 0`\nplus a `parseError` means `create` ran but printed something other than the single recipe-result JSON on\nstdout (often a stray `echo` — route it to stderr, see §10); a non-zero `exitCode` is a provider/script\nfailure described in `stderr`. Each stream is redacted and capped (head+tail) — large logs keep both the\nsetup context and the failure.\n\nThe self-test cannot see provider-side truth beyond what the scripts print, so still confirm: state has a\npopulated **authenticated** `snapshotId` (Phases 2–3 done), and `destroy` is implemented/tested (or\nexplicitly `none` — in which case the self-test won't tear down, so clean up manually).\n\nFor SSH recipes, also smoke-test the exact emitted target before declaring success: dial the host/port\nwith the identity/proxy settings, run `pwd`, verify the repo path, check the agent binary, and confirm\n`destroy` removes the provider resource/container. For Docker, inspect the auth image entrypoint and do a\nstartup-only `docker run` before the full clone/install path.\n\n---\n\n## 10. Failure modes\n\n- **Build exceeds plan timeout (e.g. Hobby 45m).** Use enough vCPUs and a timeout covering the build;\n else split work or use a higher plan. The cap also limits per-workspace runtime — surface it.\n- **Build exceeds plan RAM.** Build the **headless main only** (drop the renderer) — the biggest fitter.\n- **Private-repo clone hangs/fails.** Wrong/missing token. Use `GIT_ASKPASS` + `GIT_TERMINAL_PROMPT=0`\n so it fails fast instead of prompting.\n- **`GIT_ASKPASS` helper aborts the clone with \"`$1: unbound variable`\".** The `printf`/heredoc that writes\n the helper inside `bash -lc` under `set -u` expanded `$1`/`$GH_TOKEN` at **write** time. Escape them\n (`\\$1`, `\\$GH_TOKEN`) so they land literally and resolve at git-runtime; this also keeps the real token\n out of the file. `rm -f` the helper afterward (§5, §7f).\n- **Agent verified as \"not logged in\" despite a good login.** `codex login status` (and similar) print\n \"Logged in …\" to **stderr**; an stdout-only `grep` misses it. Prefer the status **exit code**; if you\n grep, fold stderr first (`status 2>&1 | grep …`) and match the exact success line — not `grep -qi\n'logged in'`, which also matches \"not logged in\".\n- **Headless agent login hangs.** Plain OAuth `login` starts a loopback callback server on a VM/container\n port the host browser can't reach. Use the **device-auth** flow (`login --device-auth`) — it prints a\n URL + code the user opens on the host.\n- **`known_hosts` host-key churn on local Docker.** Each ephemeral container regenerating its SSH host key\n collides on `127.0.0.1` as the published port rotates. Bake host keys into the base image at build time\n (`ssh-keygen -A`; runtime generates only if absent) so all containers share one stable key (§7h).\n- **Snapshot expired/evicted.** If `create` hits an unknown snapshot id, rerun Phases 2–3 and update\n `snapshotId`.\n- **Agent auth didn't persist.** Confirm `snapshotId` points at the **authenticated** snapshot; re-run\n Phase 3. Warn that short-lived tokens may need periodic re-auth.\n- **Agent auth copied from the host breaks.** Do not bind-mount/copy a full host agent home; sqlite\n files can be unwritable or host-specific, hooks may need approval again, and config may reference\n local-only env vars. Authenticate inside the runtime and snapshot/commit that layer.\n- **Docker auth image exits immediately.** Inspect `docker image inspect … .Config.Entrypoint` and\n `docker logs`. If the image was committed from an interactive shell, reset the entrypoint to the SSH\n entrypoint during `docker commit`.\n- **Leaked paid resource.** Every long script must trap errors and remove the sandbox it created.\n- **`create` emits non-JSON on stdout.** A stray `echo` corrupts the result — stdout is for the final\n JSON only; everything else to stderr. The `--provision` self-test surfaces this as `exitCode 0` + a\n `parseError` with the offending stdout in `provisionTranscript` (§9).\n\n---\n\n## 11. Boundaries\n\n- Don't create accounts, choose plans/regions, or invent scope/project/org/image/billing ids.\n- Don't invent or store credentials; no secrets in `userData`, state, comments, docs, or commits.\n- Don't run paid/long phases (base snapshot, auth, live test) without an explicit OK.\n- Don't hide provider errors behind generic messages — preserve actionable stderr.\n- Don't make Orca own provider lifecycle beyond invoking the configured scripts.\n- Don't commit or create an Orca workspace unless asked.\n" // oxfmt-ignore -const ORCHESTRATION_MARKDOWN = "---\nname: orchestration\ndescription: >-\n Use Orca orchestration for structured multi-agent coordination: threaded\n messages, blocking ask/reply flows, task dispatch, worker_done/escalation\n waits, task DAGs, decision gates, coordinator loops, or decomposing work\n across agents. Use `orca-cli` instead for full ownership handoffs, including\n requests phrased as \"hand off\", \"handoff\", \"handover\", \"give this to another\n agent\", or \"another worktree\" when the user did not explicitly ask to\n supervise, monitor, wait for results, or coordinate a DAG. Use `orca-cli` for\n ordinary terminal control, lightweight terminal prompts, shell commands, Orca\n worktree management, reading or waiting on terminals, and automation of the\n browser embedded inside Orca. Use Computer Use for browser windows, webviews,\n Orca app UI, or desktop UI outside Orca's embedded browser.\n---\n\n# Orca Inter-Agent Orchestration\n\nOrchestration is Orca's structured coordination layer for agent messages, task ownership, dispatch state, and worker completion tracking.\n\nUse this skill when coordination state matters. For lightweight terminal prompts or basic worktree/terminal/built-in-browser control, use `orca-cli`.\n\n## Tool Boundary\n\nIf a task says to use Orca orchestration, the coordinator must create or bind a Run, create the Task with `orca orchestration task-create`, then attach the worker with either the preferred `orca orchestration worker-start` composition or the low-level `orca orchestration dispatch --inject` path.\n\nDo not substitute non-Orca subagent tools, generic agent-spawn APIs, or chat-only parallel worker features. Those may create useful workers, but they do not create Orca task/dispatch provenance, injected lifecycle preambles, `worker_done` authority, or decision gates.\n\nBefore claiming a worker was orchestrated, verify the task/dispatch exists:\n\n```bash\norca orchestration task-list --json\norca orchestration dispatch-show --task --json\n```\n\nIf the work was accidentally run outside Orca orchestration, say so plainly. To repair provenance, rerun or revalidate the needed work through a fresh Orca terminal plus injected dispatch; do not retroactively describe the external worker as orchestrated.\n\n## When To Use\n\n- Send/reply/ask between agent terminals with persistent messages.\n- Dispatch structured tasks to workers and wait for `worker_done` or `escalation`.\n- Track task DAGs with dependencies.\n- Run coordinator loops or decision gates.\n\nDo not use orchestration merely because the user says \"hand off\", \"handoff\", \"handover\", \"give this to another agent\", or asks for another worktree/agent/model/effort. Those are full ownership transfers unless the user explicitly asks to supervise, monitor, wait for worker completion/results, coordinate a DAG, use decision gates, or keep a blocking ask/reply loop.\n\n## Preconditions\n\n- `orca status --json` should show a running runtime.\n- `orca` must be on PATH (`orca-ide` on Linux).\n- The orchestration experimental feature must be enabled in Settings > Experimental.\n- `orca orchestration` commands are RPC calls to the running Orca runtime.\n\n## Contract Migration\n\nOrca adopts a live pre-update orchestration assignment into an ordinary Run. Adoption preserves the existing agent process, PTY/session, terminal handle, tab/leaf/pane, worktree or folder workspace, Task, and Dispatch; it never restarts or replaces the worker. The retired scheduler is not revived, and a newly created attempt uses the current grammar.\n\nTreat the authority label on injected or formatted messages as definitive:\n\n- `[LEGACY COMPATIBILITY]` is live and attested. Run only the exact supported command printed with the message, using the same CLI executable and arguments that the original prompt supplied.\n- `[LEGACY RECOVERY REPLAY — MAY HAVE BEEN SEEN]` is one bounded, at-least-once cutover replay. Process it idempotently and acknowledge it only through the exact displayed guidance.\n- `[LEGACY READ-ONLY]` is inspection-only. It has no reply, acknowledgment, or lifecycle action.\n- An unlabeled current message uses the current guide and current grammar.\n\nAn explicitly selected current Run, attested current Run binding, current Dispatch, or federated attachment takes precedence over legacy fallback. A retained adoption record alone never turns a current command into a legacy call.\n\nDatabase provenance, an old-looking terminal, or a legacy Run ID does not prove mutation authority. If the runtime cannot prove liveness, principal ownership, capability, or the exact legacy contract, it degrades to read-only inspection and must not fall back to local execution. Exact recovery may restore the already-live PTY once in its original inactive background tab. It must not spawn, write, signal, stop, switch, focus, split, or inject a terminal. Loss of lifecycle authority does not invalidate the existing assignment, process, or filesystem work.\n\nCompatibility retries have narrow guarantees. A pending ask, a reply, a final Dispatch settlement, and a consuming check have durable recovery identities. A-era heartbeat and escalation calls remain at-least-once across a manual A-to-B retry because identical later signals may be intentional. If an A-era ask may already have been answered, run the exact non-consuming recovery check printed by the runtime first; after its answer is printed and acknowledged, a new invocation with the same question creates a new question. Never guess among multiple identical question threads.\n\nWhen a compatibility or recovery command returns structured next-step arguments, run those exact arguments with the same CLI executable. The arguments intentionally omit the executable name so the guidance works with `orca`, `orca-ide`, `orca-dev`, or another configured Orca CLI command. Do not translate the command from memory, broaden its recipient, or retry it as a current mutation unless the returned guidance explicitly says to.\n\nOn packaged Windows, a legacy ask uses a two-step commit/resume protocol. The initial command durably commits the question, prints its exact `ask --resume ` command, and exits with launcher status `75`; it does not wait for the answer. Run that exact resume command after the launcher or update boundary. Resume is idempotent and read-oriented: it waits for the already-committed question and does not create another one. For a WSL process that received compatibility proof at launch, use the printed executable `orca-ide` WSL resume command so the same distro and packaged launcher authority are preserved; do not substitute a PATH-resolved local CLI. Older WSL processes that never received the hidden launch token remain lifecycle read-only after the update, even while their terminal and filesystem work continue.\n\nLegacy inspection remains available without consuming mail:\n\n```bash\norca orchestration run-list --json\n# run_legacy_local is an empty audit tombstone after adoption.\norca orchestration run-show --id run_legacy_local --json\n# In run-list, find the ordinary Run whose objective is:\n# \"Recovered orchestration work from a contract update\"\norca orchestration run-show --id --json\norca orchestration task-list --run --json\norca orchestration inbox --full --json\norca orchestration check --terminal --peek --format --json\norca terminal read --terminal --json\norca terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\n```\n\nIf the original coordinator is unavailable or cannot prove its retained authority, a current coordinator may explicitly take over the adopted Run from its own live agent terminal:\n\n```bash\norca orchestration run-use --id --takeover-legacy --json\norca orchestration check --run --json\n```\n\nTakeover fences only the old coordinator, binds the current one, and moves pending worker mail into current Run Delivery. It is bound to the authenticated invoking terminal; `--from` cannot name another coordinator. Live legacy workers keep their original Tasks, Dispatches, processes, filesystems, and old prompt commands; their later questions, escalations, and completion reports route to the current coordinator. Do not use takeover while the original coordinator is still actively coordinating, because its later lifecycle mutations are rejected.\n\nDo not launch a replacement editor merely because the desktop app or runtime was updated. If adoption cannot prove continuing authority, keep the original worker as the only editor until it reaches a stable handoff point, then use a new current Dispatch in a conflict-free placement for any remaining work.\n\n## Ownership\n\nNew orchestration messages and tasks belong to one explicitly bound Run. A Run is only a durable namespace and coordinator inbox; it never schedules or places workers. Lifecycle authority comes from the active Dispatch, and terminal handles remain routing metadata rather than durable identity. Send `worker_done` and `heartbeat` from the worker's own terminal; Orca routes them to that Dispatch's Run.\n\nClassify inherited context before sending lifecycle messages:\n\n- Coordinated subtask: a live coordinator owns the DAG and waits on this dispatch. Follow the preamble exactly, including `worker_done`, heartbeat/status, `ask`, and `escalation`.\n- Full handoff means ownership transfer, not supervised dispatch. The original actor is not monitoring a DAG, so do not create lifecycle obligations unless the user explicitly asks you to supervise.\n- Classify requests containing \"hand off\", \"handoff\", \"handover\", \"give this to another agent\", \"give this to another worktree\", \"another agent\", or \"another worktree\" as full handoffs by default, even when the user names a custom model or reasoning effort.\n- Use supervised orchestration only when the user explicitly asks you to \"supervise\", \"monitor\", \"wait\", \"track completion\", \"wait for worker_done\", return results, coordinate a DAG, use a decision gate, or manage ask/reply flow.\n- Do not use `orca orchestration dispatch --inject` for full handoffs. It injects a coordinator preamble that tells the worker to send `worker_done`, heartbeat, and `ask` messages, then end its turn under the original terminal's dispatch lifecycle.\n- Do not run `orca orchestration task-create`, `orca orchestration dispatch --inject`, or `orca orchestration check --wait` for full handoffs. Do not peek at terminal output after prompt delivery to monitor progress.\n- A review-only `worker_done` reports findings; it does not authorize coordinator file edits. After a review-only completion, synthesize findings, ask a decision gate if ownership is unclear, and dispatch or hand off fixes unless the user explicitly asked the coordinator to own fixes.\n- If the user's plan names a next owner agent (for example, \"then use opencode to create a PR\"), post-review corrections and PR prep belong to that named owner. The coordinator routes, synthesizes, asks decision gates when needed, and supervises; the named owner edits files and creates the PR.\n\nIf unclear, inspect orchestration state before sending lifecycle messages:\n\n```bash\norca orchestration task-list --json\norca terminal list --json\n# If inherited context includes a task id:\norca orchestration dispatch-show --task --json\n```\n\n## Messaging\n\n```bash\norca orchestration send --subject [--to ] [--from ] [--body ] [--type ] [--priority ] [--thread-id ] [--payload ] [--json]\norca orchestration check [--terminal ] [--ack ] [--peek|--all] [--types ] [--format] [--wait] [--timeout-ms ] [--json]\norca orchestration reply --id --body [--from ] [--json]\norca orchestration ask (--question |--resume ) [--options ] [--timeout-ms ] [--from ] [--json]\norca orchestration inbox [--limit ] [--json]\n```\n\nRules:\n\n- Omit `--from` unless impersonating another terminal; Orca auto-resolves it from the current terminal.\n- A coordinator `check` returns the bound Run's oldest FIFO Delivery (up to 50 messages) and replays that exact batch until `--ack `. Process every message before acknowledging; `check --ack --wait` acknowledges, checks, and waits in one operation.\n- Use `--peek` and `--all` only for read-only history/debugging. Type filters decide when a waiter wakes; the returned actionable Delivery is still the oldest full batch.\n- Use `dispatch:` for coordinator guidance to one supervised worker. Orca routes that stable address locally or through the connected-server relay; do not substitute a remote terminal handle.\n- Terminal handles remain appropriate for low-level pre-Dispatch messaging. Prefer `agentTerminalHandle` from the create response, fall back to `startupTerminal.handle` for older runtimes, then re-resolve with `orca terminal list --worktree ... --json` if missing or stale. Continue with the replacement handle only; never dual-send to old and new handles.\n- `terminal list --json` omits `visualLayouts` because handle recovery does not need topology. Add `--include-visual-layouts` only for explicit tab and pane inspection.\n- `orca orchestration check --peek --format --json` returns locally formatted unread mail without consuming it; it never writes to terminal input or remotely wakes another terminal. Use `orchestration dispatch --inject` to deliver a tracked task, or `terminal send` when an existing agent needs a free-form prompt.\n- While supervising workers manually, use `check --wait --types worker_done,escalation,question --timeout-ms ` instead of sleep/poll loops. Process the whole Delivery, reply to `question` messages with `orca orchestration reply --id --body --json`, then acknowledge and keep waiting.\n- Treat a `check --wait` timeout or `{count:0}` as a checkpoint, not a worker failure. Long coding tasks routinely run 15-60 minutes; keep using rolling waits unless you receive `worker_done`/`escalation`, the terminal exits or disappears, or the user explicitly asks you to stop.\n- Heartbeats and visible terminal activity mean the worker is alive, not done. Do not stop, close, kill, or restart a worker just because it has not produced a completion message yet.\n- Use `ask` when a worker needs a blocking answer from the coordinator; it defaults to the active Dispatch's Run. Timeout or disconnect leaves the question pending, so resume by its original message ID instead of asking again.\n- `check --wait` returns one bounded Delivery, not every future completion. Process every message, acknowledge it, then keep waiting until every expected Dispatch settles.\n- Group addresses include `@all`, `@idle`, `@claude`, `@codex`, `@opencode`, `@gemini`, `@droid`, `@grok`, `@cursor`, and `@worktree:`.\n- Message types include `status`, `dispatch`, `worker_done`, `merge_ready`, `escalation`, `handoff`, `question`, `decision_gate` (legacy/gates), and `heartbeat`.\n- Use group addresses only for messages that are genuinely useful to many terminals, such as `status` broadcasts or intentional fan-out questions. Do not send dispatch lifecycle messages to groups.\n- `worker_done` belongs to the active Dispatch and defaults to its Run mailbox; never target a group.\n- A valid `worker_done` for the active `taskId` + `dispatchId` marks the task and dispatch completed automatically. Do not follow it with `task-update --status completed`; reserve manual updates for explicit recovery or overrides.\n- `heartbeat` is also Dispatch-scoped. Include both IDs and omit `--to` so Orca uses the owning Run; use `status` for broad progress updates.\n\n## Tasks And Dispatch\n\nA Run is the namespace/inbox, a Task is the work item, and a Dispatch assigns one Task attempt to a terminal. Create or bind a Run once before the common loop.\n\n```bash\norca orchestration run-create --objective --json\norca orchestration task-create --spec [--deps ] [--parent ] [--json]\norca orchestration task-list [--status ] [--ready] [--brief] [--json]\norca orchestration task-update --id --status [--result ] [--json]\norca orchestration dispatch --task --to [--from ] [--inject] [--json]\norca orchestration dispatch-show --task [--json]\n```\n\nTask statuses: `pending`, `ready`, `dispatched`, `completed`, `failed`, `blocked`.\n\nDispatch rules:\n\n- `--inject` sends the task spec plus preamble into a recognized agent CLI so it can report `worker_done`.\n- If the target is a bare shell, omit `--inject`, dispatch for tracking if needed, then send the prompt manually with `orca terminal send --terminal --text --enter --json`.\n- After 3 consecutive failures on one task, the dispatch context circuit-breaks and the task is marked failed.\n- Use `task-list --brief --json` for coordinator sweeps; it collapses whitespace and caps each echoed spec at 160 characters (`spec_truncated` marks shortened rows). Omit `--brief` when the full spec is required, or when an older CLI rejects it as an unknown flag.\n\n## How deep workers can nest\n\nA dispatched worker normally cannot dispatch sub-workers. Attempting it fails with\n`nested_worker_depth_exceeded` and a message telling the worker to complete the task\nitself. Do that — do not try to route around it.\n\nThe limit is a number, not an on/off switch. `Settings -> Orchestration -> Nested worker depth`\nsets how many generations are allowed:\n\n- `1` (default): a coordinator dispatches workers; those workers do not dispatch.\n- `2`: workers may dispatch one further generation.\n\nDepth is counted from the terminal that issues the command, not from the Run. Creating a\nnew Run does not reset it — a worker that runs `run-create` then `worker-start` is still a\nworker, and still counted. This is the part that changed: the old behaviour rejected\nsub-dispatch only because a worker's terminal was not bound to a Run, so creating a Run was\nenough to slip past it.\n\nTwo limits worth knowing:\n\n- **It is a guardrail, not a security boundary.** A caller that declares another terminal's\n handle while its own launch evidence is unverifiable (an ordinary restored terminal, for\n example) can be counted as that terminal instead. Orca does not treat workers as hostile.\n- **It applies while a Dispatch is active.** After `worker_done`, or after a coordinator\n settles the task, the terminal is no longer a worker and is counted as a root again. The\n process may still be alive; that is the documented boundary, not an accident.\n\n## Preferred Supervised Worker Loop\n\nUse `worker-start` for the normal supervised path. It composes the existing worktree, terminal, readiness, and dispatch primitives while returning exact created/reused effects. Agents still choose placement and concurrency; Orca does not schedule workers or infer conflicts.\n\nCreate the Run and every independent Task first, then start all independent workers before waiting:\n\n```bash\norca orchestration run-create --objective \"\" --json\norca orchestration task-create --spec \"\" --json\norca orchestration task-create --spec \"\" --json\norca orchestration worker-start --task --worktree current --agent codex --json\norca orchestration worker-start --task --worktree current --agent claude --json\n```\n\n`current` and exact existing worktrees create a fresh agent terminal and do not rerun setup. Reuse an existing agent only with `--terminal `.\n\nFor a per-invocation Claude, Codex, or Cursor launch, pass an opaque provider model id with `--model`; add `--effort` only when that agent/model supports the level. These options apply only to fresh agent terminals, override general agent default arguments, and are reported under `launch.requested` and `launch.effective` in the receipt:\n\n```bash\norca orchestration worker-start --task --worktree current --agent claude --model opus --effort high --json\n```\n\n`--effort` requires `--model`, and neither option can combine with `--terminal`. A connected worker server must advertise launch-preference support before Orca forwards either option.\n\nFor a new worktree, setup runs by default and agent-first creation reuses the returned startup agent terminal:\n\n```bash\norca orchestration worker-start --task --worktree new-child --name --agent codex --setup run --json\n# Independent/top-level:\norca orchestration worker-start --task --worktree new-top-level --name --agent codex --setup run --json\n```\n\nSetup normally starts alongside the agent. Only a repository explicitly configured with `wait-for-setup` delays agent launch until setup succeeds. Use `--setup skip` or `--setup inherit` only for a concrete reason.\n\nRead the returned receipt before continuing: `ready` plus setup `running` is normal for start-immediately, while wait-for-setup returns setup `succeeded` before accepting task input. A failed or unknown start exits nonzero; inspect its `stage`, `effects`, and `residualResources` instead of guessing or automatically retrying. A wait-for-setup timeout can honestly leave setup `running`, which is not proof of failure.\n\nTo run the worker on another connected Orca server, add `--on `. The Run and Tasks remain authoritative on the current server; later commands route by Dispatch ID, so never repeat `--on`:\n\n```bash\n# Mac Run home -> Windows worker (the reverse is identical from a Windows Run home)\norca orchestration worker-start --task --on windows --worktree new-top-level --repo --name --agent codex --setup run --json\norca orchestration worker-show --dispatch --json\norca orchestration worker-read --dispatch --limit 50 --json\norca orchestration send --to dispatch: --subject \"Follow-up\" --body \"\" --json\n```\n\nRemote `current` and `new-child` are intentionally invalid because those words are ambiguous across servers. Use an exact discovered remote worktree selector or `new-top-level` with an explicit remote repo selector.\n\nThe follow-up is structured inbox mail, not prompt injection. The worker's next\n`orchestration check` receives it even when the Dispatch is on another connected Orca server.\n\n`worker-read` defaults to `--source auto`: Orca returns the exact hook-reported Codex, Claude, OpenClaude, or Grok transcript when it can prove the worker session, otherwise it returns bounded terminal output with `source: \"terminal\"` and a typed `fallbackReason`. Continue with the returned top-level `cursor`; it stays pinned to that exact source. If Orca reports `source_changed`, start a fresh read without the old cursor. Never supply or guess a provider session ID or transcript path.\n\nWait until every expected Dispatch settles, not for a fixed number of batches:\n\n```bash\norca orchestration check --wait --types worker_done,escalation,question --timeout-ms 900000 --json\n# Process every message. For each accepted worker_done that is not immediately reused:\norca orchestration worker-release --dispatch --json\n# Acknowledge only after every message and required release decision is handled:\norca orchestration check --ack --wait --types worker_done,escalation,question --timeout-ms 900000 --json\n```\n\nAfter processing each accepted `worker_done`, choose the terminal's next owner before you acknowledge the Delivery or wait again. If the same exact agent has an immediate follow-up Task, read the `worker.agent_terminal_handle` field of `worker-show --dispatch --json`, then run `orca orchestration worker-start --task --terminal --json` so Orca transfers cleanup ownership to the new Dispatch. Otherwise run `orca orchestration worker-release --dispatch --json`.\n\nRun `worker-release` after both succeeded and failed `worker_done` reports unless the user explicitly asked to keep that worker live. Release is post-completion cleanup, not cancellation: Orca first preserves inspectable output, then closes only the exact agent terminal owned by that settled Dispatch. Reused or pre-existing terminals, setup terminals, coordinators, active workers, user-taken-over terminals, and identities Orca cannot prove are retained. If the user explicitly asks to keep the live terminal for debugging, record that exception with `orca orchestration worker-retain --dispatch --json` instead of silently skipping cleanup. When the user is finished, the same Dispatch can be passed to `worker-release`, which clears the requested retention and releases the terminal.\n\nDo not release a worker because of a timeout, TUI idle state, heartbeat, status, question, escalation, or rejected/stale `worker_done`. If release returns `release_pending` or `release_unknown`, do not substitute `terminal close`; follow the exact recovery action in the receipt. A replayed Delivery may repeat `worker-release` safely.\n\nWorkers report exactly once using the IDs and capability injected by Orca; they do not supply Run/server/terminal identity:\n\n```bash\norca orchestration send --type worker_done --subject \"\" --body \"\" --task-id --dispatch-id --outcome succeeded --files-modified \"path/a,path/b\" --json\n# On failure, use --outcome failed; never encode failure only in prose.\n```\n\nA worker question defaults to its owning Run. Timeout leaves it pending:\n\n```bash\norca orchestration ask --question \"\" --options \"yes,no\" --timeout-ms 600000 --json\norca orchestration ask --resume --timeout-ms 600000 --json\n# Coordinator:\norca orchestration reply --id --body \"\" --json\n```\n\nRecovery is conditional, never a fixed destructive sequence:\n\n- `worker-show --dispatch ` says `ready`: keep waiting or read bounded output.\n- It proves `failed` or `stopped`: start a replacement with `worker-start --task --retry-of ` plus an explicit `--on`/`--worktree` and `--agent`/`--terminal` choice. Retry does not silently inherit placement.\n- It remains `outcome_unknown`: either `worker-stop --dispatch ` and inspect again, or explicitly `worker-abandon --dispatch ` while accepting that resources may still be live. Abandon performs no remote, process, or filesystem action.\n- `worker-stop` closes only the exact supervised agent terminal. It never deletes the worktree, setup terminal, configured tabs, or unrelated processes.\n\nLow-level `worktree create`, `terminal create`, and `dispatch --inject` remain valid recipes for custom argv or topology that `worker-start` does not express.\n\n`dispatch --inject` deliberately keeps an operator-started terminal unsupervised: it never creates a `worker_dispatches` row and `worker-stop`/`worker-abandon` never close that process. The dispatch context is still authoritative, so `worker-show`, `worker-read`, and `worker-list` report it as `unsupervised`; settled `worker-retain` and `worker-release` report `retained` with `no_owned_resource` and take no process action. Use `worker-start --terminal ` when supervision and worker lifecycle state are required.\n\n## Gates And Legacy Inspection\n\n```bash\norca orchestration gate-create --task --question [--options ] [--json]\norca orchestration gate-resolve --id --resolution [--json]\norca orchestration gate-list [--task ] [--status ] [--json]\n```\n\nUse `ask` for worker-to-coordinator questions; it creates a `question` message that the coordinator answers with `reply`. Use `gate-create` only for coordinator-managed task DAG decisions, not for answering a worker's `ask`.\n\n`coordinator-start`, `coordinator-stop`, `run`, and `run-stop` are retired scheduler commands. They perform no effects and return the current-skill recovery action. They are not aliases for lightweight Run creation or binding.\n\nRecovery only: `orca orchestration reset --tasks|--messages|--all --json` clears the selected local orchestration database state. Do not run it during active coordination unless explicitly abandoning that state.\n\n## Full Handoffs\n\nFor full ownership transfer, use non-lifecycle terminal/worktree commands and then stop monitoring unless the user asks for supervision.\n\nTreat these as full handoff requests by default: \"hand off\", \"handoff\", \"handover\", \"give this to another agent\", \"give this to another worktree\", \"send this to another agent\", \"another agent\", \"another worktree\", or \"launch another agent to own this.\" Custom model or reasoning effort words such as `gpt-5.5`, `high`, or `xhigh` do not make the handoff supervised.\n\nSupervised orchestration remains available only when the user explicitly asks for supervision or coordination: \"supervise\", \"monitor\", \"wait for worker_done\", \"wait for results\", \"track completion\", \"DAG\", \"decision gate\", \"ask/reply\", or \"coordinate workers.\"\n\nDo not run `orca orchestration task-create`, `orca orchestration dispatch --inject`, or `orca orchestration check --wait` for full handoffs. `task-create` is also forbidden because it records coordinator-owned tracking state; if a task row is needed, the user asked for supervised orchestration. Do not create a `taskId`/`dispatchId`, inject a lifecycle preamble, wait for completion, or read the worker terminal after prompt delivery except to avoid losing the initial prompt.\n\nNew top-level worktree handoff:\n\n```bash\norca worktree create --name --no-parent --agent codex --prompt \"\" --setup run --json\n```\n\nBefore creating a new worktree from an active feature branch, decide and state whether the desired Orca lineage is child or top-level. Use child worktree lineage only when the new work is conceptually stacked under or dependent on the active worktree. For independent repo-wide fixes, standalone feature work, or unrelated follow-up tasks, create a top-level worktree with `--no-parent`.\n\nExisting terminal handoff:\n\n```bash\norca terminal send --terminal --text \"\" --enter --json\n```\n\nCustom Codex model/effort handoff:\n\n`orca worktree create --agent codex --prompt ...` launches the known Codex agent but does not accept Codex-specific `--model` or `-c model_reasoning_effort=...` arguments. When the user asks for a specific Codex model or effort, create the independent worktree first, launch Codex with the requested command in that worktree, wait only for TUI readiness if prompt delivery would otherwise race startup, send the prompt, and stop.\n\nThe two-step custom-argv path cannot enforce a repository's explicit `wait-for-setup` startup policy because the later `terminal create` is not the startup owned by `worktree create`. Use it only when the repository starts agents immediately. If the repository requires `wait-for-setup`, use an agent-first configured launcher that can preserve sequencing, or stop and ask rather than silently bypassing the policy.\n\nNote: when no repo default-terminal configuration supplies a primary terminal, bare create opens a fallback shell before `terminal create` adds the agent. Configured default tabs are materialized instead and may run real commands. Prefer `--agent` whenever custom argv is not required. With the two-step path, target only the agent handle; close a prior terminal only after `terminal list` or `terminal show` confirms it is an unused shell.\n\nUse the exact full `::` worktree id returned by `orca worktree create --json`; a bare repo id cannot target the new worktree.\n\n```bash\norca worktree create --name --no-parent --setup run --json\norca terminal create --worktree id: --title --command 'codex --model gpt-5.5 -c model_reasoning_effort=\"xhigh\"' --json\norca terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\norca terminal send --terminal --text \"\" --enter --json\n```\n\nWait only for `tui-idle` when needed to avoid losing the prompt. Do not monitor task completion.\n\n`--no-parent` only controls Orca lineage; it does not choose the Git base. If the work should start from the repo default base, omit `--base-branch` so Orca uses that default, or explicitly pass the repo default base (`origin/main`, `origin/master`, or the `orca repo show --repo --json` value); never base it on the current feature branch unless the user explicitly asks for stacked work or \"branch from current\". Put current-branch context in the prompt instead.\n\n## Worker Terminals\n\nChoose the worker location before creating a terminal. `Fresh worker` means a fresh agent session, not a new git worktree. For parallel work, create one fresh agent terminal per worker in the same required worktree, falling back to the active worktree when none is named. If the task says current worktree only, depends on uncommitted files/artifacts, or must validate/PR the current branch, keep every worker in the active worktree:\n\n```bash\norca terminal create --worktree active --title --command \"codex\" --json\norca terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\norca orchestration dispatch --task --to --inject --json\n```\n\nReuse an idle agent in the required worktree only if the prompt allows reuse; otherwise create a fresh terminal there. Create a new worktree only when the user explicitly requests one or a concrete checkout or filesystem conflict makes sharing unsafe or impossible; if the user did not request it, state that conflict before running `worktree create`. Independent tasks, parallel execution, convenience, or a preference for separate checkouts are not isolation requirements.\n\nWhen a new worktree is allowed, use child lineage for isolated work that is stacked under or dependent on the active worktree, and use `--no-parent` when it is not stacked. Decide the Git base separately: `--no-parent` makes the worktree top-level in Orca, while omitted `--base-branch` uses the repo default base.\n\nFor every new worktree, pass `--setup run` so any configured repository setup hook runs. This does not mean waiting for setup before agent launch: preserve the repository's startup policy, whose default starts setup and the agent side by side. Use `--setup skip` or `--setup inherit` only when there is a concrete task-specific reason, and state that reason before creating the worktree. This rule does not rerun setup for current or existing worktrees.\n\n```bash\norca worktree create --name --agent codex --setup run --json\n# or: --agent claude | omp | pi | grok | ...\n# Read from agentTerminalHandle, falling back to startupTerminal.handle.\norca terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\norca orchestration dispatch --task --to --inject --json\n```\n\nFor new-worktree workers, read the id and `agentTerminalHandle` from `worktree create`, falling back to `startupTerminal.handle` for older runtimes. Use that as the sole worker handle when present; otherwise use `terminal list` to resolve the agent handle. Omit `--repo` only inside an Orca-managed worktree; otherwise pass `--repo `.\n\n**For an allowed new worktree, use agent-first:** `--agent` reveals the new worktree and launches the selected agent **in its first terminal**, without adding a separate fallback shell for that worker. Pass `--setup run`; repo setup and default-terminal settings may add intentional tabs or splits. Do **not** run bare `worktree create` and then `terminal create --command ` for the same worker when agent-first create is available: without configured default tabs, that two-step path leaves a fallback shell + agent pair. Only use it when custom agent argv is required (for example Codex model/effort flags) or when an older CLI rejects `--agent`; if you must, message only the agent handle. Configured default tabs are intentional surfaces, so close a prior terminal only after `terminal list` or `terminal show` confirms it is an unused shell. Do not run `worktree create` when the task must stay in the current worktree.\n\nUse `orca worktree create --prompt ...` or `orca terminal send ...` for full handoffs or untracked/lightweight prompts. Those paths do not attach `taskId`/`dispatchId`; the worker should not send lifecycle messages unless the prompt supplies a live orchestration preamble.\n\nSidebar lineage and orchestration lifecycle are related but not identical. A same-worktree worker may appear as a peer under that worktree in the sidebar while remaining a child dispatch in orchestration state; only an actual child worktree creates visible parent/child worktree lineage.\n\nOther terminal commands coordinators often need:\n\n```bash\norca terminal list [--worktree ] [--include-visual-layouts] [--json]\norca terminal create [--worktree ] [--title ] [--command ] [--json]\norca terminal split --terminal [--direction horizontal|vertical] [--command ] [--json]\norca terminal wait --terminal --for tui-idle --timeout-ms --json\norca terminal read --terminal --json\norca terminal send --terminal --text --enter --json\n```\n\nIf an older CLI rejects `worktree create --agent`, create the worktree normally, then run `orca terminal create --worktree --command \"codex\" --json` or `--command \"claude\"`.\n\nWait for `tui-idle` before dispatching. Always pass `--timeout-ms`; real coding tasks can take 15-60 minutes. During supervision, use rolling `check --wait` windows. If a window returns no matching message, inspect `task-list`, `terminal read`, or `terminal wait --for tui-idle` as a liveness checkpoint; if the terminal is still working or producing activity, keep waiting instead of retrying the task.\n\n## Agent Guidance\n\n- Workers with a valid live preamble must send `worker_done` exactly once from their own terminal with an explicit `--outcome succeeded` or `--outcome failed`:\n `orca orchestration send --type worker_done --subject \"\" --body \"<3-sentence summary: what you did, what you found, what's left>\" --task-id --dispatch-id --outcome succeeded --files-modified \"path/a\" --report-path \"\" --json`\n- A failed outcome is still a terminal report, but Orca records both the Dispatch and Task as failed. Never encode failure only in the subject/body.\n- After sending `worker_done`, end that dispatched turn and idle at the agent prompt. Do not autonomously start more work, poll, or attempt to close the terminal yourself. A direct user instruction takes precedence and starts ordinary user-owned work: follow it without coordinator approval or a fresh Dispatch, never refuse it because of worker/coordinator roles, and do not reuse the settled Dispatch's lifecycle IDs. A coordinator-supervised follow-up still arrives with a fresh preamble + TASK block.\n- For long tasks, send heartbeat/status only when the preamble asks for it, including both IDs:\n `orca orchestration send --type heartbeat --subject \"alive\" --payload '{\"taskId\":\"\",\"dispatchId\":\"\",\"phase\":\"implementing\"}' --json`\n- If blocked before completion, use `ask`; use `escalation` only when ownership is valid and the coordinator must intervene.\n- Treat preambles inherited through terminal history or full handoffs as stale unless the current prompt explicitly keeps that coordinator in the loop.\n- Coordinators must account for every settled worker terminal before waiting again or ending the turn: immediately reuse the exact worker for a new Dispatch, explicitly retain it at the user's request with `worker-retain`, or run `worker-release`. Do not leave a completed worker live merely to inspect output; released workers remain readable through `worker-read`.\n- Coordinators should use `task-list --ready` as external memory, dispatch parallel waves, and avoid dependency chains deeper than 3-4 steps.\n\n## Example\n\n```bash\norca terminal create --worktree active --title login-css-worker --command \"claude\" --json\norca terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\norca orchestration task-create --spec \"Fix the login button CSS\" --json\norca orchestration dispatch --task --to --inject --json\norca orchestration check --wait --types worker_done,escalation,question --timeout-ms 900000 --json\n```\n\n## Next Action\n\nCoordinator: confirm `orca status --json`, create or bind a Run, inspect `task-list`/`dispatch-show` if inheriting state, then use the explicit supervised loop (`task-create` -> `worker-start` -> `check --wait`). Use low-level terminal creation plus `dispatch --inject` only when the composed start does not express the needed topology. After every accepted `worker_done`, either transfer the exact terminal to an immediate follow-up Dispatch or run `worker-release` before the next wait.\n\nWorker: if the current prompt contains a live dispatch preamble, do the task, use `ask` for blocking questions, and send `worker_done` once with the required payload. If the preamble is stale or absent, do not send lifecycle messages; inspect state or treat the prompt as an ordinary handoff.\n" +const ORCHESTRATION_MARKDOWN = "---\nname: orchestration\ndescription: >-\n Use Orca orchestration for structured multi-agent coordination: threaded\n messages, blocking ask/reply flows, task dispatch, worker_done/escalation\n waits, task DAGs, decision gates, coordinator loops, or decomposing work\n across agents. Use `orca-cli` instead for full ownership handoffs, including\n requests phrased as \"hand off\", \"handoff\", \"handover\", \"give this to another\n agent\", or \"another worktree\" when the user did not explicitly ask to\n supervise, monitor, wait for results, or coordinate a DAG. Use `orca-cli` for\n ordinary terminal control, lightweight terminal prompts, shell commands, Orca\n worktree management, reading or waiting on terminals, and automation of the\n browser embedded inside Orca. Use Computer Use for browser windows, webviews,\n Orca app UI, or desktop UI outside Orca's embedded browser.\n---\n\n# Orca Inter-Agent Orchestration\n\nOrchestration is Orca's structured coordination layer for agent messages, task ownership, dispatch state, and worker completion tracking.\n\nUse this skill when coordination state matters. For lightweight terminal prompts or basic worktree/terminal/built-in-browser control, use `orca-cli`.\n\n## Tool Boundary\n\nIf a task says to use Orca orchestration, the coordinator must create or bind a Run, create the Task with `orca orchestration task-create`, then attach the worker with either the preferred `orca orchestration worker-start` composition or the low-level `orca orchestration dispatch --inject` path.\n\nDo not substitute non-Orca subagent tools, generic agent-spawn APIs, or chat-only parallel worker features. Those may create useful workers, but they do not create Orca task/dispatch provenance, injected lifecycle preambles, `worker_done` authority, or decision gates.\n\nBefore claiming a worker was orchestrated, verify the task/dispatch exists:\n\n```bash\norca orchestration task-list --json\norca orchestration dispatch-show --task --json\n```\n\nIf the work was accidentally run outside Orca orchestration, say so plainly. To repair provenance, rerun or revalidate the needed work through a fresh Orca terminal plus injected dispatch; do not retroactively describe the external worker as orchestrated.\n\n## When To Use\n\n- Send/reply/ask between agent terminals with persistent messages.\n- Dispatch structured tasks to workers and wait for `worker_done` or `escalation`.\n- Track task DAGs with dependencies.\n- Run coordinator loops or decision gates.\n\nDo not use orchestration merely because the user says \"hand off\", \"handoff\", \"handover\", \"give this to another agent\", or asks for another worktree/agent/model/effort. Those are full ownership transfers unless the user explicitly asks to supervise, monitor, wait for worker completion/results, coordinate a DAG, use decision gates, or keep a blocking ask/reply loop.\n\n## Preconditions\n\n- `orca status --json` should show a running runtime.\n- `orca` must be on PATH (`orca-ide` on Linux).\n- The orchestration experimental feature must be enabled in Settings > Experimental.\n- `orca orchestration` commands are RPC calls to the running Orca runtime.\n\n## Contract Migration\n\nOrca adopts a live pre-update orchestration assignment into an ordinary Run. Adoption preserves the existing agent process, PTY/session, terminal handle, tab/leaf/pane, worktree or folder workspace, Task, and Dispatch; it never restarts or replaces the worker. The retired scheduler is not revived, and a newly created attempt uses the current grammar.\n\nTreat the authority label on injected or formatted messages as definitive:\n\n- `[LEGACY COMPATIBILITY]` is live and attested. Run only the exact supported command printed with the message, using the same CLI executable and arguments that the original prompt supplied.\n- `[LEGACY RECOVERY REPLAY — MAY HAVE BEEN SEEN]` is one bounded, at-least-once cutover replay. Process it idempotently and acknowledge it only through the exact displayed guidance.\n- `[LEGACY READ-ONLY]` is inspection-only. It has no reply, acknowledgment, or lifecycle action.\n- An unlabeled current message uses the current guide and current grammar.\n\nAn explicitly selected current Run, attested current Run binding, current Dispatch, or federated attachment takes precedence over legacy fallback. A retained adoption record alone never turns a current command into a legacy call.\n\nDatabase provenance, an old-looking terminal, or a legacy Run ID does not prove mutation authority. If the runtime cannot prove liveness, principal ownership, capability, or the exact legacy contract, it degrades to read-only inspection and must not fall back to local execution. Exact recovery may restore the already-live PTY once in its original inactive background tab. It must not spawn, write, signal, stop, switch, focus, split, or inject a terminal. Loss of lifecycle authority does not invalidate the existing assignment, process, or filesystem work.\n\nCompatibility retries have narrow guarantees. A pending ask, a reply, a final Dispatch settlement, and a consuming check have durable recovery identities. A-era heartbeat and escalation calls remain at-least-once across a manual A-to-B retry because identical later signals may be intentional. If an A-era ask may already have been answered, run the exact non-consuming recovery check printed by the runtime first; after its answer is printed and acknowledged, a new invocation with the same question creates a new question. Never guess among multiple identical question threads.\n\nWhen a compatibility or recovery command returns structured next-step arguments, run those exact arguments with the same CLI executable. The arguments intentionally omit the executable name so the guidance works with `orca`, `orca-ide`, `orca-dev`, or another configured Orca CLI command. Do not translate the command from memory, broaden its recipient, or retry it as a current mutation unless the returned guidance explicitly says to.\n\nOn packaged Windows, a legacy ask uses a two-step commit/resume protocol. The initial command durably commits the question, prints its exact `ask --resume ` command, and exits with launcher status `75`; it does not wait for the answer. Run that exact resume command after the launcher or update boundary. Resume is idempotent and read-oriented: it waits for the already-committed question and does not create another one. For a WSL process that received compatibility proof at launch, use the printed executable `orca-ide` WSL resume command so the same distro and packaged launcher authority are preserved; do not substitute a PATH-resolved local CLI. Older WSL processes that never received the hidden launch token remain lifecycle read-only after the update, even while their terminal and filesystem work continue.\n\nLegacy inspection remains available without consuming mail:\n\n```bash\norca orchestration run-list --json\n# run_legacy_local is an empty audit tombstone after adoption.\norca orchestration run-show --id run_legacy_local --json\n# In run-list, find the ordinary Run whose objective is:\n# \"Recovered orchestration work from a contract update\"\norca orchestration run-show --id --json\norca orchestration task-list --run --json\norca orchestration inbox --full --json\norca orchestration check --terminal --peek --format --json\norca terminal read --terminal --json\norca terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\n```\n\nIf the original coordinator is unavailable or cannot prove its retained authority, a current coordinator may explicitly take over the adopted Run from its own live agent terminal:\n\n```bash\norca orchestration run-use --id --takeover-legacy --json\norca orchestration check --run --json\n```\n\nTakeover fences only the old coordinator, binds the current one, and moves pending worker mail into current Run Delivery. It is bound to the authenticated invoking terminal; `--from` cannot name another coordinator. Live legacy workers keep their original Tasks, Dispatches, processes, filesystems, and old prompt commands; their later questions, escalations, and completion reports route to the current coordinator. Do not use takeover while the original coordinator is still actively coordinating, because its later lifecycle mutations are rejected.\n\nDo not launch a replacement editor merely because the desktop app or runtime was updated. If adoption cannot prove continuing authority, keep the original worker as the only editor until it reaches a stable handoff point, then use a new current Dispatch in a conflict-free placement for any remaining work.\n\n## Run Coordinator Authority\n\nOrca orchestration is agent-operated. A Run therefore has one current coordinator agent, not a human operator. That coordinator is the sole task-graph writer and consuming mailbox reader for coordinator-side operations; workers retain only their exact Dispatch capabilities for heartbeat, questions, escalation, and `worker_done`. This is per-Run authority, not an operating-system or global Orca privilege.\n\nThe single-writer rule prevents two agents from changing Task, Dispatch, worker, or gate state concurrently and prevents two consumers from acknowledging different views of the same FIFO Delivery. The current coordinator may create and update Tasks (`task-create`, `task-update`), start or dispatch workers (`worker-start`, `dispatch`), create and resolve gates (`gate-create`, `gate-resolve`), answer worker questions (`reply`), and consume or acknowledge Run mail (`check`). Explicit `run-show`, `task-list --run`, `gate-list --run`, and inbox calls remain read-only for non-owners and headless callers. `check --peek` is non-consuming, but an ordinary current Run still requires its coordinator authority; do not use it as a non-owner inspection path.\n\n`run-use` is an authority claim, not a read-only selection. When a transfer is intended, make one claim from the stable replacement agent terminal with `orca orchestration run-use --id --json`; do not guess whether its process is a remint or spoof the owner with `--from`. Orca either preserves same-process authority, grants replacement authority after proving exit, or rejects the claim with `effectsApplied: false`.\n\nOn rejection, read `error.data.coordinatorStatus`, `claimantStatus`, `nextSteps`, and any exact command-argument fields. `coordinatorStatus` is `live` or `unverifiable` because a proven `exited` incumbent permits the claim to succeed. If `claimantStatus` is `changed`, the invoking agent changed during proof; follow the returned retry arguments once from one stable process instead of treating it as a network verdict or retrying blindly. Never infer authority from a terminal handle alone.\n\n- Same coordinator process, reminted handle: authority and any outstanding Delivery are preserved without advancing the consumer generation.\n- Different process and `live` incumbent: `consumer_fenced`; continue from the owning coordinator terminal. To transfer intentionally, stop or exit the owning coordinator process before retrying from its replacement.\n- Different process and `unverifiable` incumbent: `consumer_fenced`; restore connectivity to the owning host. Loss of contact is never evidence of exit, including SSH, relay, Windows, WSL, and federated runtimes.\n- Different process and `exited` incumbent: the replacement may run `orca orchestration run-use --id --json`. Orca advances the consumer generation, fences the old outstanding Delivery, preserves pending Run mail and worker assignments, and grants the replacement coordinator authority.\n\nInspect without claiming authority:\n\n```bash\norca orchestration run-show --id --json\norca orchestration task-list --run --json\n```\n\n`binding.currentConsumer` is `true` only for the current coordinator. `false` permits inspection, not coordinator mutations or consuming `check` calls, and never proves that the Run is unowned.\n\nNo live-to-live transfer command exists. A seamless live handoff would require a separate owner-authorized protocol; do not simulate one by retrying, changing `--from`, or replacing a terminal handle. `--takeover-legacy` is not a force override for ordinary Runs: it is limited to the automatically adopted legacy Run described above.\n\n## Ownership\n\nNew orchestration messages and tasks belong to one explicitly bound Run. A Run is only a durable namespace and coordinator inbox; it never schedules or places workers. Lifecycle authority comes from the active Dispatch, and terminal handles remain routing metadata rather than durable identity. Send `worker_done` and `heartbeat` from the worker's own terminal; Orca routes them to that Dispatch's Run.\n\nClassify inherited context before sending lifecycle messages:\n\n- Coordinated subtask: a live coordinator owns the DAG and waits on this dispatch. Follow the preamble exactly, including `worker_done`, heartbeat/status, `ask`, and `escalation`.\n- Full handoff means ownership transfer, not supervised dispatch. The original actor is not monitoring a DAG, so do not create lifecycle obligations unless the user explicitly asks you to supervise.\n- Classify requests containing \"hand off\", \"handoff\", \"handover\", \"give this to another agent\", \"give this to another worktree\", \"another agent\", or \"another worktree\" as full handoffs by default, even when the user names a custom model or reasoning effort.\n- Use supervised orchestration only when the user explicitly asks you to \"supervise\", \"monitor\", \"wait\", \"track completion\", \"wait for worker_done\", return results, coordinate a DAG, use a decision gate, or manage ask/reply flow.\n- Do not use `orca orchestration dispatch --inject` for full handoffs. It injects a coordinator preamble that tells the worker to send `worker_done`, heartbeat, and `ask` messages, then end its turn under the original terminal's dispatch lifecycle.\n- Do not run `orca orchestration task-create`, `orca orchestration dispatch --inject`, or `orca orchestration check --wait` for full handoffs. Do not peek at terminal output after prompt delivery to monitor progress.\n- A review-only `worker_done` reports findings; it does not authorize coordinator file edits. After a review-only completion, synthesize findings, ask a decision gate if ownership is unclear, and dispatch or hand off fixes unless the user explicitly asked the coordinator to own fixes.\n- If the user's plan names a next owner agent (for example, \"then use opencode to create a PR\"), post-review corrections and PR prep belong to that named owner. The coordinator routes, synthesizes, asks decision gates when needed, and supervises; the named owner edits files and creates the PR.\n\nIf unclear, inspect orchestration state before sending lifecycle messages:\n\n```bash\norca orchestration task-list --json\norca terminal list --json\n# If inherited context includes a task id:\norca orchestration dispatch-show --task --json\n```\n\n## Messaging\n\n```bash\norca orchestration send --subject [--to ] [--from ] [--body ] [--type ] [--priority ] [--thread-id ] [--payload ] [--json]\norca orchestration check [--terminal ] [--ack ] [--peek|--all] [--types ] [--format] [--wait] [--timeout-ms ] [--json]\norca orchestration reply --id --body [--from ] [--json]\norca orchestration ask (--question |--resume ) [--options ] [--timeout-ms ] [--from ] [--json]\norca orchestration inbox [--limit ] [--json]\n```\n\nRules:\n\n- Omit `--from` unless an injected preamble or exact recovery command supplies it; Orca normally resolves the current terminal. A declared handle is routing input, never proof of coordinator authority or a transfer mechanism.\n- A coordinator `check` returns the bound Run's oldest FIFO Delivery (up to 50 messages) and replays that exact batch until `--ack `. Process every message before acknowledging; `check --ack --wait` acknowledges, checks, and waits in one operation.\n- Use `--peek` and `--all` only for read-only history/debugging. Type filters decide when a waiter wakes; the returned actionable Delivery is still the oldest full batch.\n- Use `dispatch:` for coordinator guidance to one supervised worker. Orca routes that stable address locally or through the connected-server relay; do not substitute a remote terminal handle.\n- Terminal handles remain appropriate for low-level pre-Dispatch messaging. Prefer `agentTerminalHandle` from the create response, fall back to `startupTerminal.handle` for older runtimes, then re-resolve with `orca terminal list --worktree ... --json` if missing or stale. Continue with the replacement handle only; never dual-send to old and new handles.\n- `terminal list --json` omits `visualLayouts` because handle recovery does not need topology. Add `--include-visual-layouts` only for explicit tab and pane inspection.\n- `orca orchestration check --peek --format --json` returns locally formatted unread mail without consuming it; it never writes to terminal input or remotely wakes another terminal. Use `orchestration dispatch --inject` to deliver a tracked task, or `terminal send` when an existing agent needs a free-form prompt.\n- While supervising workers manually, use `check --wait --types worker_done,escalation,question --timeout-ms ` instead of sleep/poll loops. Process the whole Delivery, reply to `question` messages with `orca orchestration reply --id --body --json`, then acknowledge and keep waiting.\n- Treat a `check --wait` timeout or `{count:0}` as a checkpoint, not a worker failure. Long coding tasks routinely run 15-60 minutes; keep using rolling waits unless you receive `worker_done`/`escalation`, the terminal exits or disappears, or the user explicitly asks you to stop.\n- Heartbeats and visible terminal activity mean the worker is alive, not done. Do not stop, close, kill, or restart a worker just because it has not produced a completion message yet.\n- Use `ask` when a worker needs a blocking answer from the coordinator; it defaults to the active Dispatch's Run. Timeout or disconnect leaves the question pending, so resume by its original message ID instead of asking again.\n- `check --wait` returns one bounded Delivery, not every future completion. Process every message, acknowledge it, then keep waiting until every expected Dispatch settles.\n- Group addresses include `@all`, `@idle`, `@claude`, `@codex`, `@opencode`, `@gemini`, `@droid`, `@grok`, `@cursor`, and `@worktree:`.\n- Message types include `status`, `dispatch`, `worker_done`, `merge_ready`, `escalation`, `handoff`, `question`, `decision_gate` (legacy/gates), and `heartbeat`.\n- Use group addresses only for messages that are genuinely useful to many terminals, such as `status` broadcasts or intentional fan-out questions. Do not send dispatch lifecycle messages to groups.\n- `worker_done` belongs to the active Dispatch and defaults to its Run mailbox; never target a group.\n- A valid `worker_done` for the active `taskId` + `dispatchId` marks the task and dispatch completed automatically. Do not follow it with `task-update --status completed`; reserve manual updates for explicit recovery or overrides.\n- `heartbeat` is also Dispatch-scoped. Include both IDs and omit `--to` so Orca uses the owning Run; use `status` for broad progress updates.\n\n## Tasks And Dispatch\n\nA Run is the namespace/inbox, a Task is the work item, and a Dispatch assigns one Task attempt to a terminal. Create or bind a Run once before the common loop.\n\n```bash\norca orchestration run-create --objective --json\norca orchestration task-create --spec [--deps ] [--parent ] [--json]\norca orchestration task-list [--status ] [--ready] [--brief] [--json]\norca orchestration task-update --id --status [--result ] [--json]\norca orchestration dispatch --task --to [--from ] [--inject] [--json]\norca orchestration dispatch-show --task [--json]\n```\n\nTask statuses: `pending`, `ready`, `dispatched`, `completed`, `failed`, `blocked`.\n\nDispatch rules:\n\n- `--inject` sends the task spec plus preamble into a recognized agent CLI so it can report `worker_done`.\n- If the target is a bare shell, omit `--inject`, dispatch for tracking if needed, then send the prompt manually with `orca terminal send --terminal --text --enter --json`.\n- After 3 consecutive failures on one task, the dispatch context circuit-breaks and the task is marked failed.\n- Use `task-list --brief --json` for coordinator sweeps; it collapses whitespace and caps each echoed spec at 160 characters (`spec_truncated` marks shortened rows). Omit `--brief` when the full spec is required, or when an older CLI rejects it as an unknown flag.\n\n## How deep workers can nest\n\nA dispatched worker normally cannot dispatch sub-workers. Attempting it fails with\n`nested_worker_depth_exceeded` and a message telling the worker to complete the task\nitself. Do that — do not try to route around it.\n\nThe limit is a number, not an on/off switch. `Settings -> Orchestration -> Nested worker depth`\nsets how many generations are allowed:\n\n- `1` (default): a coordinator dispatches workers; those workers do not dispatch.\n- `2`: workers may dispatch one further generation.\n\nDepth is counted from the terminal that issues the command, not from the Run. Creating a\nnew Run does not reset it — a worker that runs `run-create` then `worker-start` is still a\nworker, and still counted. This is the part that changed: the old behaviour rejected\nsub-dispatch only because a worker's terminal was not bound to a Run, so creating a Run was\nenough to slip past it.\n\nTwo limits worth knowing:\n\n- **It is a guardrail, not a security boundary.** A caller that declares another terminal's\n handle while its own launch evidence is unverifiable (an ordinary restored terminal, for\n example) can be counted as that terminal instead. This affects only nesting-depth\n classification; it never grants coordinator or Dispatch authority. Orca does not treat\n workers as hostile.\n- **It applies while a Dispatch is active.** After `worker_done`, or after a coordinator\n settles the task, the terminal is no longer a worker and is counted as a root again. The\n process may still be alive; that is the documented boundary, not an accident.\n\n## Preferred Supervised Worker Loop\n\nUse `worker-start` for the normal supervised path. It composes the existing worktree, terminal, readiness, and dispatch primitives while returning exact created/reused effects. Agents still choose placement and concurrency; Orca does not schedule workers or infer conflicts.\n\nCreate the Run and every independent Task first, then start all independent workers before waiting:\n\n```bash\norca orchestration run-create --objective \"\" --json\norca orchestration task-create --spec \"\" --json\norca orchestration task-create --spec \"\" --json\norca orchestration worker-start --task --worktree current --agent codex --json\norca orchestration worker-start --task --worktree current --agent claude --json\n```\n\n`current` and exact existing worktrees create a fresh agent terminal and do not rerun setup. Reuse an existing agent only with `--terminal `.\n\nFor a per-invocation Claude, Codex, or Cursor launch, pass an opaque provider model id with `--model`; add `--effort` only when that agent/model supports the level. These options apply only to fresh agent terminals, override general agent default arguments, and are reported under `launch.requested` and `launch.effective` in the receipt:\n\n```bash\norca orchestration worker-start --task --worktree current --agent claude --model opus --effort high --json\n```\n\n`--effort` requires `--model`, and neither option can combine with `--terminal`. A connected worker server must advertise launch-preference support before Orca forwards either option.\n\nFor a new worktree, setup runs by default and agent-first creation reuses the returned startup agent terminal:\n\n```bash\norca orchestration worker-start --task --worktree new-child --name --agent codex --setup run --json\n# Independent/top-level:\norca orchestration worker-start --task --worktree new-top-level --name --agent codex --setup run --json\n```\n\nSetup normally starts alongside the agent. Only a repository explicitly configured with `wait-for-setup` delays agent launch until setup succeeds. Use `--setup skip` or `--setup inherit` only for a concrete reason.\n\nRead the returned receipt before continuing: `ready` plus setup `running` is normal for start-immediately, while wait-for-setup returns setup `succeeded` before accepting task input. A failed or unknown start exits nonzero; inspect its `stage`, `effects`, and `residualResources` instead of guessing or automatically retrying. A wait-for-setup timeout can honestly leave setup `running`, which is not proof of failure.\n\nTo run the worker on another connected Orca server, add `--on `. The Run and Tasks remain authoritative on the current server; later commands route by Dispatch ID, so never repeat `--on`:\n\n```bash\n# Mac Run home -> Windows worker (the reverse is identical from a Windows Run home)\norca orchestration worker-start --task --on windows --worktree new-top-level --repo --name --agent codex --setup run --json\norca orchestration worker-show --dispatch --json\norca orchestration worker-read --dispatch --limit 50 --json\norca orchestration send --to dispatch: --subject \"Follow-up\" --body \"\" --json\n```\n\nRemote `current` and `new-child` are intentionally invalid because those words are ambiguous across servers. Use an exact discovered remote worktree selector or `new-top-level` with an explicit remote repo selector.\n\nThe follow-up is structured inbox mail, not prompt injection. The worker's next\n`orchestration check` receives it even when the Dispatch is on another connected Orca server.\n\n`worker-read` defaults to `--source auto`: Orca returns the exact hook-reported Codex, Claude, OpenClaude, or Grok transcript when it can prove the worker session, otherwise it returns bounded terminal output with `source: \"terminal\"` and a typed `fallbackReason`. Continue with the returned top-level `cursor`; it stays pinned to that exact source. If Orca reports `source_changed`, start a fresh read without the old cursor. Never supply or guess a provider session ID or transcript path.\n\nWait until every expected Dispatch settles, not for a fixed number of batches:\n\n```bash\norca orchestration check --wait --types worker_done,escalation,question --timeout-ms 900000 --json\n# Process every message. For each accepted worker_done that is not immediately reused:\norca orchestration worker-release --dispatch --json\n# Acknowledge only after every message and required release decision is handled:\norca orchestration check --ack --wait --types worker_done,escalation,question --timeout-ms 900000 --json\n```\n\nAfter processing each accepted `worker_done`, choose the terminal's next owner before you acknowledge the Delivery or wait again. If the same exact agent has an immediate follow-up Task, read the `worker.agent_terminal_handle` field of `worker-show --dispatch --json`, then run `orca orchestration worker-start --task --terminal --json` so Orca transfers cleanup ownership to the new Dispatch. Otherwise run `orca orchestration worker-release --dispatch --json`.\n\nRun `worker-release` after both succeeded and failed `worker_done` reports unless the user explicitly asked to keep that worker live. Release is post-completion cleanup, not cancellation: Orca first preserves inspectable output, then closes only the exact agent terminal owned by that settled Dispatch. Reused or pre-existing terminals, setup terminals, coordinators, active workers, user-taken-over terminals, and identities Orca cannot prove are retained. If the user explicitly asks to keep the live terminal for debugging, record that exception with `orca orchestration worker-retain --dispatch --json` instead of silently skipping cleanup. When the user is finished, the same Dispatch can be passed to `worker-release`, which clears the requested retention and releases the terminal.\n\nDo not release a worker because of a timeout, TUI idle state, heartbeat, status, question, escalation, or rejected/stale `worker_done`. If release returns `release_pending` or `release_unknown`, do not substitute `terminal close`; follow the exact recovery action in the receipt. A replayed Delivery may repeat `worker-release` safely.\n\nWorkers report exactly once using the IDs and capability injected by Orca; they do not supply Run/server/terminal identity:\n\n```bash\norca orchestration send --type worker_done --subject \"\" --body \"\" --task-id --dispatch-id --outcome succeeded --files-modified \"path/a,path/b\" --json\n# On failure, use --outcome failed; never encode failure only in prose.\n```\n\nA worker question defaults to its owning Run. Timeout leaves it pending:\n\n```bash\norca orchestration ask --question \"\" --options \"yes,no\" --timeout-ms 600000 --json\norca orchestration ask --resume --timeout-ms 600000 --json\n# Coordinator:\norca orchestration reply --id --body \"\" --json\n```\n\nRecovery is conditional, never a fixed destructive sequence:\n\n- `worker-show --dispatch ` says `ready`: keep waiting or read bounded output.\n- It proves `failed` or `stopped`: start a replacement with `worker-start --task --retry-of ` plus an explicit `--on`/`--worktree` and `--agent`/`--terminal` choice. Retry does not silently inherit placement.\n- It remains `outcome_unknown`: either `worker-stop --dispatch ` and inspect again, or explicitly `worker-abandon --dispatch ` while accepting that resources may still be live. Abandon performs no remote, process, or filesystem action.\n- `worker-stop` closes only the exact supervised agent terminal. It never deletes the worktree, setup terminal, configured tabs, or unrelated processes.\n\nLow-level `worktree create`, `terminal create`, and `dispatch --inject` remain valid recipes for custom argv or topology that `worker-start` does not express.\n\n`dispatch --inject` deliberately keeps an operator-started terminal unsupervised: it never creates a `worker_dispatches` row and `worker-stop`/`worker-abandon` never close that process. The dispatch context is still authoritative, so `worker-show`, `worker-read`, and `worker-list` report it as `unsupervised`; settled `worker-retain` and `worker-release` report `retained` with `no_owned_resource` and take no process action. Use `worker-start --terminal ` when supervision and worker lifecycle state are required.\n\n## Gates And Legacy Inspection\n\n```bash\norca orchestration gate-create --task --question [--options ] [--json]\norca orchestration gate-resolve --id --resolution [--json]\norca orchestration gate-list [--task ] [--status ] [--json]\n```\n\nUse `ask` for worker-to-coordinator questions; it creates a `question` message that the coordinator answers with `reply`. Use `gate-create` only for coordinator-managed task DAG decisions, not for answering a worker's `ask`.\n\n`coordinator-start`, `coordinator-stop`, `run`, and `run-stop` are retired scheduler commands. They perform no effects and return the current-skill recovery action. They are not aliases for lightweight Run creation or binding.\n\nRecovery only: `orca orchestration reset --tasks|--messages|--all --json` clears the selected local orchestration database state. Do not run it during active coordination unless explicitly abandoning that state.\n\n## Full Handoffs\n\nFor full ownership transfer, use non-lifecycle terminal/worktree commands and then stop monitoring unless the user asks for supervision.\n\nTreat these as full handoff requests by default: \"hand off\", \"handoff\", \"handover\", \"give this to another agent\", \"give this to another worktree\", \"send this to another agent\", \"another agent\", \"another worktree\", or \"launch another agent to own this.\" Custom model or reasoning effort words such as `gpt-5.5`, `high`, or `xhigh` do not make the handoff supervised.\n\nSupervised orchestration remains available only when the user explicitly asks for supervision or coordination: \"supervise\", \"monitor\", \"wait for worker_done\", \"wait for results\", \"track completion\", \"DAG\", \"decision gate\", \"ask/reply\", or \"coordinate workers.\"\n\nDo not run `orca orchestration task-create`, `orca orchestration dispatch --inject`, or `orca orchestration check --wait` for full handoffs. `task-create` is also forbidden because it records coordinator-owned tracking state; if a task row is needed, the user asked for supervised orchestration. Do not create a `taskId`/`dispatchId`, inject a lifecycle preamble, wait for completion, or read the worker terminal after prompt delivery except to avoid losing the initial prompt.\n\nNew top-level worktree handoff:\n\n```bash\norca worktree create --name --no-parent --agent codex --prompt \"\" --setup run --json\n```\n\nBefore creating a new worktree from an active feature branch, decide and state whether the desired Orca lineage is child or top-level. Use child worktree lineage only when the new work is conceptually stacked under or dependent on the active worktree. For independent repo-wide fixes, standalone feature work, or unrelated follow-up tasks, create a top-level worktree with `--no-parent`.\n\nExisting terminal handoff:\n\n```bash\norca terminal send --terminal --text \"\" --enter --json\n```\n\nCustom Codex model/effort handoff:\n\n`orca worktree create --agent codex --prompt ...` launches the known Codex agent but does not accept Codex-specific `--model` or `-c model_reasoning_effort=...` arguments. When the user asks for a specific Codex model or effort, create the independent worktree first, launch Codex with the requested command in that worktree, wait only for TUI readiness if prompt delivery would otherwise race startup, send the prompt, and stop.\n\nThe two-step custom-argv path cannot enforce a repository's explicit `wait-for-setup` startup policy because the later `terminal create` is not the startup owned by `worktree create`. Use it only when the repository starts agents immediately. If the repository requires `wait-for-setup`, use an agent-first configured launcher that can preserve sequencing, or stop and ask rather than silently bypassing the policy.\n\nNote: when no repo default-terminal configuration supplies a primary terminal, bare create opens a fallback shell before `terminal create` adds the agent. Configured default tabs are materialized instead and may run real commands. Prefer `--agent` whenever custom argv is not required. With the two-step path, target only the agent handle; close a prior terminal only after `terminal list` or `terminal show` confirms it is an unused shell.\n\nUse the exact full `::` worktree id returned by `orca worktree create --json`; a bare repo id cannot target the new worktree.\n\n```bash\norca worktree create --name --no-parent --setup run --json\norca terminal create --worktree id: --title --command 'codex --model gpt-5.5 -c model_reasoning_effort=\"xhigh\"' --json\norca terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\norca terminal send --terminal --text \"\" --enter --json\n```\n\nWait only for `tui-idle` when needed to avoid losing the prompt. Do not monitor task completion.\n\n`--no-parent` only controls Orca lineage; it does not choose the Git base. If the work should start from the repo default base, omit `--base-branch` so Orca uses that default, or explicitly pass the repo default base (`origin/main`, `origin/master`, or the `orca repo show --repo --json` value); never base it on the current feature branch unless the user explicitly asks for stacked work or \"branch from current\". Put current-branch context in the prompt instead.\n\n## Worker Terminals\n\nChoose the worker location before creating a terminal. `Fresh worker` means a fresh agent session, not a new git worktree. For parallel work, create one fresh agent terminal per worker in the same required worktree, falling back to the active worktree when none is named. If the task says current worktree only, depends on uncommitted files/artifacts, or must validate/PR the current branch, keep every worker in the active worktree:\n\n```bash\norca terminal create --worktree active --title --command \"codex\" --json\norca terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\norca orchestration dispatch --task --to --inject --json\n```\n\nReuse an idle agent in the required worktree only if the prompt allows reuse; otherwise create a fresh terminal there. Create a new worktree only when the user explicitly requests one or a concrete checkout or filesystem conflict makes sharing unsafe or impossible; if the user did not request it, state that conflict before running `worktree create`. Independent tasks, parallel execution, convenience, or a preference for separate checkouts are not isolation requirements.\n\nWhen a new worktree is allowed, use child lineage for isolated work that is stacked under or dependent on the active worktree, and use `--no-parent` when it is not stacked. Decide the Git base separately: `--no-parent` makes the worktree top-level in Orca, while omitted `--base-branch` uses the repo default base.\n\nFor every new worktree, pass `--setup run` so any configured repository setup hook runs. This does not mean waiting for setup before agent launch: preserve the repository's startup policy, whose default starts setup and the agent side by side. Use `--setup skip` or `--setup inherit` only when there is a concrete task-specific reason, and state that reason before creating the worktree. This rule does not rerun setup for current or existing worktrees.\n\n```bash\norca worktree create --name --agent codex --setup run --json\n# or: --agent claude | omp | pi | grok | ...\n# Read from agentTerminalHandle, falling back to startupTerminal.handle.\norca terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\norca orchestration dispatch --task --to --inject --json\n```\n\nFor new-worktree workers, read the id and `agentTerminalHandle` from `worktree create`, falling back to `startupTerminal.handle` for older runtimes. Use that as the sole worker handle when present; otherwise use `terminal list` to resolve the agent handle. Omit `--repo` only inside an Orca-managed worktree; otherwise pass `--repo `.\n\n**For an allowed new worktree, use agent-first:** `--agent` reveals the new worktree and launches the selected agent **in its first terminal**, without adding a separate fallback shell for that worker. Pass `--setup run`; repo setup and default-terminal settings may add intentional tabs or splits. Do **not** run bare `worktree create` and then `terminal create --command ` for the same worker when agent-first create is available: without configured default tabs, that two-step path leaves a fallback shell + agent pair. Only use it when custom agent argv is required (for example Codex model/effort flags) or when an older CLI rejects `--agent`; if you must, message only the agent handle. Configured default tabs are intentional surfaces, so close a prior terminal only after `terminal list` or `terminal show` confirms it is an unused shell. Do not run `worktree create` when the task must stay in the current worktree.\n\nUse `orca worktree create --prompt ...` or `orca terminal send ...` for full handoffs or untracked/lightweight prompts. Those paths do not attach `taskId`/`dispatchId`; the worker should not send lifecycle messages unless the prompt supplies a live orchestration preamble.\n\nSidebar lineage and orchestration lifecycle are related but not identical. A same-worktree worker may appear as a peer under that worktree in the sidebar while remaining a child dispatch in orchestration state; only an actual child worktree creates visible parent/child worktree lineage.\n\nOther terminal commands coordinators often need:\n\n```bash\norca terminal list [--worktree ] [--include-visual-layouts] [--json]\norca terminal create [--worktree ] [--title ] [--command ] [--json]\norca terminal split --terminal [--direction horizontal|vertical] [--command ] [--json]\norca terminal wait --terminal --for tui-idle --timeout-ms --json\norca terminal read --terminal --json\norca terminal send --terminal --text --enter --json\n```\n\nIf an older CLI rejects `worktree create --agent`, create the worktree normally, then run `orca terminal create --worktree --command \"codex\" --json` or `--command \"claude\"`.\n\nWait for `tui-idle` before dispatching. Always pass `--timeout-ms`; real coding tasks can take 15-60 minutes. During supervision, use rolling `check --wait` windows. If a window returns no matching message, inspect `task-list`, `terminal read`, or `terminal wait --for tui-idle` as a liveness checkpoint; if the terminal is still working or producing activity, keep waiting instead of retrying the task.\n\n## Agent Guidance\n\n- Workers with a valid live preamble must send `worker_done` exactly once from their own terminal with an explicit `--outcome succeeded` or `--outcome failed`:\n `orca orchestration send --type worker_done --subject \"\" --body \"<3-sentence summary: what you did, what you found, what's left>\" --task-id --dispatch-id --outcome succeeded --files-modified \"path/a\" --report-path \"\" --json`\n- A failed outcome is still a terminal report, but Orca records both the Dispatch and Task as failed. Never encode failure only in the subject/body.\n- After sending `worker_done`, end that dispatched turn and idle at the agent prompt. Do not autonomously start more work, poll, or attempt to close the terminal yourself. A direct user instruction takes precedence and starts ordinary user-owned work: follow it without coordinator approval or a fresh Dispatch, never refuse it because of worker/coordinator roles, and do not reuse the settled Dispatch's lifecycle IDs. A coordinator-supervised follow-up still arrives with a fresh preamble + TASK block.\n- For long tasks, send heartbeat/status only when the preamble asks for it, including both IDs:\n `orca orchestration send --type heartbeat --subject \"alive\" --payload '{\"taskId\":\"\",\"dispatchId\":\"\",\"phase\":\"implementing\"}' --json`\n- If blocked before completion, use `ask`; use `escalation` only when ownership is valid and the coordinator must intervene.\n- Treat preambles inherited through terminal history or full handoffs as stale unless the current prompt explicitly keeps that coordinator in the loop.\n- Coordinators must account for every settled worker terminal before waiting again or ending the turn: immediately reuse the exact worker for a new Dispatch, explicitly retain it at the user's request with `worker-retain`, or run `worker-release`. Do not leave a completed worker live merely to inspect output; released workers remain readable through `worker-read`.\n- Coordinators should use `task-list --ready` as external memory, dispatch parallel waves, and avoid dependency chains deeper than 3-4 steps.\n\n## Example\n\n```bash\norca terminal create --worktree active --title login-css-worker --command \"claude\" --json\norca terminal wait --terminal --for tui-idle --timeout-ms 60000 --json\norca orchestration task-create --spec \"Fix the login button CSS\" --json\norca orchestration dispatch --task --to --inject --json\norca orchestration check --wait --types worker_done,escalation,question --timeout-ms 900000 --json\n```\n\n## Next Action\n\nCoordinator: confirm `orca status --json`, create or bind a Run, inspect `task-list`/`dispatch-show` if inheriting state, then use the explicit supervised loop (`task-create` -> `worker-start` -> `check --wait`). Use low-level terminal creation plus `dispatch --inject` only when the composed start does not express the needed topology. After every accepted `worker_done`, either transfer the exact terminal to an immediate follow-up Dispatch or run `worker-release` before the next wait.\n\nWorker: if the current prompt contains a live dispatch preamble, do the task, use `ask` for blocking questions, and send `worker_done` once with the required payload. If the preamble is stale or absent, do not send lifecycle messages; inspect state or treat the prompt as an ordinary handoff.\n" // Why: no current guide has bundled reference documents, so --full is byte-identical for now. // oxfmt-ignore diff --git a/src/cli/handlers/orchestration-gate-cli.test.ts b/src/cli/handlers/orchestration-gate-cli.test.ts index b4be315c155d..fba3c90c46c5 100644 --- a/src/cli/handlers/orchestration-gate-cli.test.ts +++ b/src/cli/handlers/orchestration-gate-cli.test.ts @@ -158,7 +158,7 @@ describe('orchestration gate commands carry caller identity', () => { ) }) - it('inspects a named Run without resolving a caller terminal', async () => { + it('inspects a named Run without requiring a caller terminal', async () => { // Why: read-only inspection must stay reachable from a pane with no bound Run. getTerminalHandleMock.mockRejectedValue( new RuntimeClientError('no_active_terminal', 'no active terminal') @@ -167,17 +167,22 @@ describe('orchestration gate commands carry caller identity', () => { callMock, okFixture('req_list', { gates: [{ id: 'gate_1', task_id: 'task_1', question: 'ship?', status: 'pending' }], - count: 1 + count: 1, + runId: 'run_adopted', + binding: { currentConsumer: false } }) ) - await main(['orchestration', 'gate-list', '--run', 'run_adopted', '--json'], '/tmp/repo') + await main(['orchestration', 'gate-list', '--run', 'run_adopted'], '/tmp/repo') expect(process.exitCode).toBe(0) - expect(getTerminalHandleMock).not.toHaveBeenCalled() + expect(getTerminalHandleMock).toHaveBeenCalled() expect(paramsFor('orchestration.gateList')).toEqual( expect.objectContaining({ run: 'run_adopted', from: undefined }) ) + expect(logSpy.mock.calls.map((call) => String(call[0])).join('\n')).toContain( + 'listed read-only' + ) }) it('fails an unbound gate-create with an actionable error and no mutation', async () => { diff --git a/src/cli/handlers/orchestration-run-cli.test.ts b/src/cli/handlers/orchestration-run-cli.test.ts index 7159e8024903..984369982121 100644 --- a/src/cli/handlers/orchestration-run-cli.test.ts +++ b/src/cli/handlers/orchestration-run-cli.test.ts @@ -119,6 +119,75 @@ describe('lightweight Run CLI handlers', () => { takeoverLegacy: true }) }) + + it('discloses a non-owner run-show as read-only', async () => { + const response = { + result: { + run: { + id: 'run_1', + objective: 'Inspect work', + consumer_generation: 1, + legacy: 0, + created_at: '2026-08-27 20:00:00' + }, + binding: { currentConsumer: false } + } + } + callMock.mockResolvedValue(response) + vi.mocked(printResult).mockClear() + + await ORCHESTRATION_HANDLERS['orchestration run-show']({ + flags: new Map([['id', 'run_1']]), + client: { call: callMock }, + cwd: '/tmp/repo', + json: false + } as never) + + expect(callMock).toHaveBeenCalledWith('orchestration.runShow', { + id: 'run_1', + from: 'term_coord' + }) + const format = vi.mocked(printResult).mock.calls[0]?.[2] as (result: { + run: { + id: string + objective: string + consumer_generation: number + legacy: number + created_at: string + } + binding: { currentConsumer: boolean } + }) => string + expect(format(response.result)).toContain('shown read-only') + }) + + it('discloses headless run-show inspection as read-only', async () => { + delete process.env.ORCA_TERMINAL_HANDLE + getTerminalHandleMock.mockRejectedValue({ code: 'no_active_terminal' }) + callMock.mockResolvedValue({ + result: { + run: { + id: 'run_1', + objective: 'Inspect headlessly', + consumer_generation: 1, + legacy: 0, + created_at: '2026-08-27 20:00:00' + }, + binding: { currentConsumer: false } + } + }) + + await ORCHESTRATION_HANDLERS['orchestration run-show']({ + flags: new Map([['id', 'run_1']]), + client: { call: callMock }, + cwd: '/tmp/repo', + json: true + } as never) + + expect(callMock).toHaveBeenCalledWith('orchestration.runShow', { + id: 'run_1', + from: undefined + }) + }) }) describe('orchestration reset CLI handler', () => { @@ -174,6 +243,69 @@ describe('orchestration reset CLI handler', () => { }) describe('orchestration task-list brief output', () => { + it('keeps explicit Run inspection available without an active terminal', async () => { + delete process.env.ORCA_TERMINAL_HANDLE + getTerminalHandleMock.mockRejectedValue({ code: 'no_active_terminal' }) + callMock.mockReset().mockResolvedValue({ + result: { + tasks: [], + count: 0, + runId: 'run_1', + binding: { currentConsumer: false } + } + }) + + await ORCHESTRATION_HANDLERS['orchestration task-list']({ + flags: new Map([['run', 'run_1']]), + client: { call: callMock }, + cwd: '/tmp/repo', + json: true + } as never) + + expect(callMock).toHaveBeenCalledWith( + 'orchestration.taskList', + expect.objectContaining({ run: 'run_1', callerTerminalHandle: undefined }) + ) + }) + + it('discloses non-owner inspection even when the Run has no tasks', async () => { + process.env.ORCA_TERMINAL_HANDLE = 'term_inspector' + callMock.mockReset().mockResolvedValue({ + result: { + tasks: [], + count: 0, + runId: 'run_1', + binding: { currentConsumer: false } + } + }) + vi.mocked(printResult).mockClear() + + await ORCHESTRATION_HANDLERS['orchestration task-list']({ + flags: new Map([['run', 'run_1']]), + client: { call: callMock }, + json: false + } as never) + + expect(callMock).toHaveBeenCalledWith( + 'orchestration.taskList', + expect.objectContaining({ run: 'run_1', callerTerminalHandle: 'term_inspector' }) + ) + const format = vi.mocked(printResult).mock.calls[0]?.[2] as (result: { + count: number + runId: string + tasks: never[] + binding: { currentConsumer: boolean } + }) => string + expect( + format({ + count: 0, + runId: 'run_1', + tasks: [], + binding: { currentConsumer: false } + }) + ).toContain('not bound to this terminal') + }) + it('requests server-side brief and falls back client-side for older runtimes', async () => { callMock.mockReset().mockResolvedValue({ result: { diff --git a/src/cli/handlers/orchestration-terminal-identity.test.ts b/src/cli/handlers/orchestration-terminal-identity.test.ts new file mode 100644 index 000000000000..8aba130c1d1f --- /dev/null +++ b/src/cli/handlers/orchestration-terminal-identity.test.ts @@ -0,0 +1,45 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { RuntimeClientError } from '../runtime-client' +import { resolveOrchestrationTerminalHandle } from './orchestration/terminal-identity' + +const originalTerminalHandle = process.env.ORCA_TERMINAL_HANDLE +const originalPaneKey = process.env.ORCA_PANE_KEY + +afterEach(() => { + restoreEnv('ORCA_TERMINAL_HANDLE', originalTerminalHandle) + restoreEnv('ORCA_PANE_KEY', originalPaneKey) +}) + +describe('orchestration terminal identity', () => { + it('refreshes caller evidence after resolving a stale handle by pane', async () => { + process.env.ORCA_TERMINAL_HANDLE = 'term_stale' + process.env.ORCA_PANE_KEY = 'tab_1:leaf_1' + const call = vi + .fn() + .mockRejectedValueOnce(new RuntimeClientError('terminal_handle_stale', 'stale')) + .mockResolvedValueOnce({ result: { terminal: { handle: 'term_reminted' } } }) + const refresh = vi.fn() + + const handle = await resolveOrchestrationTerminalHandle( + new Map(), + '/tmp/repo', + { + call, + refreshOrchestrationCallerHandleAfterPaneRemint: refresh + } as never, + 'from', + { validateEnvHandle: true } + ) + + expect(handle).toBe('term_reminted') + expect(refresh).toHaveBeenCalledWith('term_stale', 'tab_1:leaf_1', 'term_reminted') + }) +}) + +function restoreEnv(name: string, value: string | undefined): void { + if (value === undefined) { + delete process.env[name] + } else { + process.env[name] = value + } +} diff --git a/src/cli/handlers/orchestration/gate-handlers.ts b/src/cli/handlers/orchestration/gate-handlers.ts index 27d3cc1e3b76..5113448e429c 100644 --- a/src/cli/handlers/orchestration/gate-handlers.ts +++ b/src/cli/handlers/orchestration/gate-handlers.ts @@ -2,7 +2,10 @@ import type { CommandHandler } from '../../dispatch' import { printResult } from '../../format' import { getOptionalStringFlag, getRequiredStringFlag } from '../../flags' import { callOrchestrationMutation } from './mutation-request' -import { resolveCoordinatorTerminalHandle } from './terminal-identity' +import { + resolveCoordinatorTerminalHandle, + resolveOptionalCoordinatorTerminalHandle +} from './terminal-identity' export const ORCHESTRATION_GATE_HANDLERS: Record = { 'orchestration gate-create': async ({ flags, client, cwd, json }) => { @@ -37,11 +40,14 @@ export const ORCHESTRATION_GATE_HANDLERS: Record = { 'orchestration gate-list': async ({ flags, client, cwd, json }) => { const run = getOptionalStringFlag(flags, 'run') // Why: named runs remain inspectable without a pane; only implicit runs resolve identity. - const from = run ? undefined : await resolveCoordinatorTerminalHandle(flags, cwd, client) + const from = run + ? await resolveOptionalCoordinatorTerminalHandle(flags, cwd, client) + : await resolveCoordinatorTerminalHandle(flags, cwd, client) const result = await client.call<{ gates: { id: string; task_id: string; question: string; status: string }[] count: number runId?: string + binding?: { currentConsumer: boolean } }>('orchestration.gateList', { task: getOptionalStringFlag(flags, 'task'), status: getOptionalStringFlag(flags, 'status'), @@ -49,12 +55,17 @@ export const ORCHESTRATION_GATE_HANDLERS: Record = { from }) printResult(result, json, (value) => { + const nonOwnerNotice = + value.binding?.currentConsumer === false + ? `Run ${value.runId} is not bound to this terminal; listed read-only. Mutations require the owning coordinator.` + : undefined if (value.gates.length === 0) { - return 'No gates found.' + return nonOwnerNotice ? `${nonOwnerNotice}\nNo gates found.` : 'No gates found.' } - return value.gates + const gates = value.gates .map((gate) => `${gate.id} task=${gate.task_id} [${gate.status}] "${gate.question}"`) .join('\n') + return nonOwnerNotice ? `${nonOwnerNotice}\n${gates}` : gates }) } } diff --git a/src/cli/handlers/orchestration/run-handlers.ts b/src/cli/handlers/orchestration/run-handlers.ts index 7c77913df158..4e2d9052f951 100644 --- a/src/cli/handlers/orchestration/run-handlers.ts +++ b/src/cli/handlers/orchestration/run-handlers.ts @@ -7,7 +7,10 @@ import { } from '../../flags' import { ORCHESTRATION_RUN_PAGE_LIMIT } from '../../../shared/orchestration-run-pagination' import { callOrchestrationMutation } from './mutation-request' -import { resolveCoordinatorTerminalHandle } from './terminal-identity' +import { + resolveCoordinatorTerminalHandle, + resolveOptionalCoordinatorTerminalHandle +} from './terminal-identity' export const ORCHESTRATION_RUN_HANDLERS: Record = { 'orchestration run-create': async ({ flags, client, cwd, json }) => { @@ -64,7 +67,8 @@ export const ORCHESTRATION_RUN_HANDLERS: Record = { }) }, - 'orchestration run-show': async ({ flags, client, json }) => { + 'orchestration run-show': async ({ flags, client, cwd, json }) => { + const runId = getRequiredStringFlag(flags, 'id') const result = await client.call<{ run: { id: string @@ -73,12 +77,20 @@ export const ORCHESTRATION_RUN_HANDLERS: Record = { legacy: number created_at: string } - }>('orchestration.runShow', { id: getRequiredStringFlag(flags, 'id') }) + binding?: { currentConsumer: boolean } + }>('orchestration.runShow', { + id: runId, + from: await resolveOptionalCoordinatorTerminalHandle(flags, cwd, client) + }) printResult( result, json, (r) => - `${r.run.id}${r.run.legacy ? ' [legacy, inspect only]' : ''} ${r.run.objective}\n` + + `${ + r.binding?.currentConsumer === false + ? `Run ${r.run.id} is not bound to this terminal; shown read-only. Mutations require the owning coordinator.\n` + : '' + }${r.run.id}${r.run.legacy ? ' [legacy, inspect only]' : ''} ${r.run.objective}\n` + `consumer generation ${r.run.consumer_generation}; created ${r.run.created_at}` ) } diff --git a/src/cli/handlers/orchestration/task-handlers.ts b/src/cli/handlers/orchestration/task-handlers.ts index c4c943261e8c..87d00a33eda7 100644 --- a/src/cli/handlers/orchestration/task-handlers.ts +++ b/src/cli/handlers/orchestration/task-handlers.ts @@ -4,7 +4,10 @@ import { getOptionalStringFlag, getRequiredStringFlag } from '../../flags' import { RuntimeClientError } from '../../runtime-client' import { abbreviateOrchestrationTasks } from '../../../shared/orchestration-task-summary' import { callOrchestrationMutation } from './mutation-request' -import { resolveCoordinatorTerminalHandle } from './terminal-identity' +import { + resolveCoordinatorTerminalHandle, + resolveOptionalCoordinatorTerminalHandle +} from './terminal-identity' const TASK_STATUS_VALUES = [ 'pending', @@ -39,7 +42,7 @@ export const ORCHESTRATION_TASK_HANDLERS: Record = { const brief = flags.has('brief') const run = getOptionalStringFlag(flags, 'run') const callerTerminalHandle = run - ? undefined + ? await resolveOptionalCoordinatorTerminalHandle(flags, cwd, client) : await resolveCoordinatorTerminalHandle(flags, cwd, client) const result = await client.call<{ tasks: { @@ -55,6 +58,7 @@ export const ORCHESTRATION_TASK_HANDLERS: Record = { count: number runId?: string legacyReadOnly?: boolean + binding?: { currentConsumer: boolean } }>('orchestration.taskList', { status: getOptionalStringFlag(flags, 'status'), ready: flags.has('ready') ? true : undefined, @@ -72,8 +76,15 @@ export const ORCHESTRATION_TASK_HANDLERS: Record = { } : result printResult(output, json, (r) => { + const nonOwnerNotice = + r.binding?.currentConsumer === false + ? `Run ${r.runId} is not bound to this terminal; listed read-only. Mutations require the owning coordinator.` + : undefined if (r.count === 0) { - return r.legacyReadOnly ? 'No legacy tasks (read-only).' : 'No tasks.' + if (r.legacyReadOnly) { + return 'No legacy tasks (read-only).' + } + return nonOwnerNotice ? `${nonOwnerNotice}\nNo tasks.` : 'No tasks.' } const tasks = r.tasks .map((task) => { @@ -85,7 +96,10 @@ export const ORCHESTRATION_TASK_HANDLERS: Record = { return head }) .join('\n') - return r.legacyReadOnly ? `Legacy Run ${r.runId} (read-only)\n${tasks}` : tasks + if (r.legacyReadOnly) { + return `Legacy Run ${r.runId} (read-only)\n${tasks}` + } + return nonOwnerNotice ? `${nonOwnerNotice}\n${tasks}` : tasks }) }, diff --git a/src/cli/handlers/orchestration/terminal-identity.ts b/src/cli/handlers/orchestration/terminal-identity.ts index 5da11afac1d1..90f2d8e916ff 100644 --- a/src/cli/handlers/orchestration/terminal-identity.ts +++ b/src/cli/handlers/orchestration/terminal-identity.ts @@ -77,7 +77,16 @@ async function resolveOrchestrationPaneTerminalHandle( const response = await client.call<{ terminal: { handle: string } }>('terminal.resolvePane', { paneKey }) - return response.result.terminal.handle + const remintedHandle = response.result.terminal.handle + const previousHandle = process.env.ORCA_TERMINAL_HANDLE + if (previousHandle) { + client.refreshOrchestrationCallerHandleAfterPaneRemint?.( + previousHandle, + paneKey, + remintedHandle + ) + } + return remintedHandle } catch (err) { if ( isPaneRemintUnavailableError(err) || @@ -127,6 +136,21 @@ export async function resolveCoordinatorTerminalHandle( }) } +export async function resolveOptionalCoordinatorTerminalHandle( + flags: Map, + cwd: string, + client: RuntimeClient +): Promise { + try { + return await resolveCoordinatorTerminalHandle(flags, cwd, client) + } catch (err) { + if (getClientErrorCode(err) === 'no_active_sender_terminal') { + return undefined + } + throw err + } +} + async function resolveImplicitOrchestrationSender( flags: Map, cwd: string, diff --git a/src/cli/index.test.ts b/src/cli/index.test.ts index 1b190d9dbde1..6a37bfaca33a 100644 --- a/src/cli/index.test.ts +++ b/src/cli/index.test.ts @@ -291,6 +291,24 @@ describe('orca root help', () => { logSpy.mockRestore() }) + it('describes Run authority claims and retired coordinator commands for agents', async () => { + const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}) + + await main(['--help'], '/tmp/repo') + + const help = logSpy.mock.calls.flat().join('\n') + expect(help).toContain( + "orchestration run-use Claim an existing Run's coordinator authority" + ) + expect(help).toContain( + 'orchestration coordinator-start Retired: load the current orchestration skill' + ) + expect(help).toContain( + 'orchestration coordinator-stop Retired: load the current orchestration skill' + ) + logSpy.mockRestore() + }) + it('advertises host-local account management', async () => { const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}) diff --git a/src/cli/root-help-text-primary.ts b/src/cli/root-help-text-primary.ts index 6f282c884fda..4994358a7821 100644 --- a/src/cli/root-help-text-primary.ts +++ b/src/cli/root-help-text-primary.ts @@ -93,7 +93,7 @@ export const ROOT_HELP_TEXT_PRIMARY = [ '', 'Orchestration:', ' orchestration run-create Create and bind a lightweight orchestration Run', - ' orchestration run-use Bind this coordinator terminal to an existing Run', + " orchestration run-use Claim an existing Run's coordinator authority", " orchestration run-current Show this terminal's bound Run", ' orchestration run-list List lightweight orchestration Runs', ' orchestration run-show Show one lightweight orchestration Run', @@ -115,8 +115,8 @@ export const ROOT_HELP_TEXT_PRIMARY = [ " orchestration worker-release Release a settled worker's terminal after archiving its output", ' orchestration worker-retain Keep a worker terminal live for debugging', ' orchestration worker-list Report worker terminal resource accounting', - ' orchestration coordinator-start Start the legacy automatic coordinator loop', - ' orchestration coordinator-stop Stop the legacy automatic coordinator loop', + ' orchestration coordinator-start Retired: load the current orchestration skill', + ' orchestration coordinator-stop Retired: load the current orchestration skill', ' orchestration gate-create Create a decision gate blocking a task', ' orchestration gate-resolve Resolve a pending decision gate', ' orchestration gate-list List decision gates', diff --git a/src/cli/runtime-client.test.ts b/src/cli/runtime-client.test.ts index 2fa6130ce3d7..07e71b8102e3 100644 --- a/src/cli/runtime-client.test.ts +++ b/src/cli/runtime-client.test.ts @@ -55,6 +55,14 @@ function writeMetadata( ) } +function restoreEnv(name: string, value: string | undefined): void { + if (value === undefined) { + delete process.env[name] + } else { + process.env[name] = value + } +} + function findUnusedPid(seed = 200_000): number { // Why: the stale-bootstrap test must point metadata at a definitely-dead // process. Hard-coding a small PID is host-dependent and flakes when that @@ -139,6 +147,62 @@ describe.skipIf(process.platform === 'win32')('RuntimeClient', () => { expect(requests[3]?.compatibilityInvocationId).not.toBe(requests[1]?.compatibilityInvocationId) }) + it('refreshes caller evidence after a pane-proven terminal handle remint', async () => { + const userDataPath = mkdtempSync(join(tmpdir(), 'orca-runtime-client-')) + const endpoint = join(userDataPath, 'runtime.sock') + const requests: Record[] = [] + const server = createServer((socket) => { + sockets.add(socket) + socket.once('close', () => sockets.delete(socket)) + socket.once('data', (data) => { + const request = JSON.parse(String(data).trim()) as Record + requests.push(request) + const result = + request.method === 'status.get' + ? { capabilities: [ORCHESTRATION_CONTRACT_RUNTIME_CAPABILITY] } + : {} + socket.write( + `${JSON.stringify({ + id: request.id, + ok: true, + result, + _meta: { runtimeId: 'runtime-1' } + })}\n` + ) + }) + }) + servers.add(server) + await new Promise((resolve) => server.listen(endpoint, resolve)) + writeMetadata(userDataPath, endpoint) + + const priorHandle = process.env.ORCA_TERMINAL_HANDLE + const priorPaneKey = process.env.ORCA_PANE_KEY + const priorLaunchToken = process.env.ORCA_AGENT_LAUNCH_TOKEN + process.env.ORCA_TERMINAL_HANDLE = 'term_stale' + process.env.ORCA_PANE_KEY = 'tab_1:leaf_1' + process.env.ORCA_AGENT_LAUNCH_TOKEN = 'launch-secret' + const client = new RuntimeClient(userDataPath, 500) + try { + client.refreshOrchestrationCallerHandleAfterPaneRemint( + 'term_stale', + 'tab_1:leaf_1', + 'term_reminted' + ) + await client.call('orchestration.taskList', {}) + } finally { + restoreEnv('ORCA_TERMINAL_HANDLE', priorHandle) + restoreEnv('ORCA_PANE_KEY', priorPaneKey) + restoreEnv('ORCA_AGENT_LAUNCH_TOKEN', priorLaunchToken) + } + + expect(requests[0]?.method).toBe('orchestration.taskList') + expect(requests[0]?.orchestrationCompatibilityEvidence).toEqual({ + terminalHandle: 'term_reminted', + paneKey: 'tab_1:leaf_1', + launchToken: 'launch-secret' + }) + }) + it('rejects an old local runtime before sending an orchestration mutation', async () => { const userDataPath = mkdtempSync(join(tmpdir(), 'orca-runtime-client-')) const endpoint = join(userDataPath, 'runtime.sock') diff --git a/src/cli/runtime/client.ts b/src/cli/runtime/client.ts index f9a855ae9299..5a3303b24f70 100644 --- a/src/cli/runtime/client.ts +++ b/src/cli/runtime/client.ts @@ -30,6 +30,7 @@ import { buildOrchestrationRecoveryCommand, resolveOrchestrationCliExecutable } from './orchestration-recovery-command' +import { refreshOrchestrationCallerHandleAfterPaneRemint } from './orchestration-caller-handle-remint' // Why: for long-poll methods the caller's method-level // `params.timeoutMs` is the inner waiter budget; we extend the client-side @@ -55,9 +56,7 @@ export class RuntimeClient { private readonly originalArgs: readonly string[] | undefined private readonly remoteCompat: RemoteRuntimeCompatGate private orchestrationContractCheck: Promise | null = null - private readonly orchestrationCompatibility = createOrchestrationCompatibilityEnvelope( - process.env - ) + private orchestrationCompatibility = createOrchestrationCompatibilityEnvelope(process.env) // Why: browser commands trigger first-time session init (agent-browser connect + // CDP proxy setup) which can take 15-30s. 60s accommodates cold start without @@ -83,6 +82,19 @@ export class RuntimeClient { return this.remotePairing !== null } + refreshOrchestrationCallerHandleAfterPaneRemint( + previousHandle: string, + paneKey: string, + remintedHandle: string + ): void { + this.orchestrationCompatibility = refreshOrchestrationCallerHandleAfterPaneRemint( + this.orchestrationCompatibility, + previousHandle, + paneKey, + remintedHandle + ) + } + async call( method: string, params?: unknown, diff --git a/src/cli/runtime/orchestration-caller-handle-remint.ts b/src/cli/runtime/orchestration-caller-handle-remint.ts new file mode 100644 index 000000000000..5880a1f5ed34 --- /dev/null +++ b/src/cli/runtime/orchestration-caller-handle-remint.ts @@ -0,0 +1,22 @@ +import type { RuntimeOrchestrationEnvelope } from '../../shared/runtime-rpc-envelope' + +export function refreshOrchestrationCallerHandleAfterPaneRemint( + envelope: RuntimeOrchestrationEnvelope, + previousHandle: string, + paneKey: string, + remintedHandle: string +): RuntimeOrchestrationEnvelope { + const evidence = envelope.orchestrationCompatibilityEvidence + if ( + !evidence || + evidence.terminalHandle !== previousHandle || + evidence.paneKey !== paneKey || + !remintedHandle + ) { + return envelope + } + return { + ...envelope, + orchestrationCompatibilityEvidence: { ...evidence, terminalHandle: remintedHandle } + } +} diff --git a/src/cli/specs/orchestration.test.ts b/src/cli/specs/orchestration.test.ts new file mode 100644 index 000000000000..e93c87ad2c5a --- /dev/null +++ b/src/cli/specs/orchestration.test.ts @@ -0,0 +1,20 @@ +import { describe, expect, it } from 'vitest' +import { ORCHESTRATION_COMMAND_SPECS } from './orchestration' + +describe('orchestration command specs', () => { + it('describes run-use as a fenced authority claim for agents', () => { + const runUse = ORCHESTRATION_COMMAND_SPECS.find( + (spec) => spec.path.join(' ') === 'orchestration run-use' + ) + const guidance = [runUse?.summary, ...(runUse?.notes ?? [])].join('\n') + + expect(guidance).toContain('coordinator authority') + expect(guidance).toContain('owning host proves the prior coordinator exited') + expect(guidance).toContain('live') + expect(guidance).toContain('unverifiable') + expect(guidance).toContain('consumer_fenced') + expect(guidance).toContain('Do not retry blindly') + expect(guidance).toContain('--takeover-legacy') + expect(guidance).toContain('not a force override for ordinary Runs') + }) +}) diff --git a/src/cli/specs/orchestration.ts b/src/cli/specs/orchestration.ts index f266e93b10a0..8624817f2bc7 100644 --- a/src/cli/specs/orchestration.ts +++ b/src/cli/specs/orchestration.ts @@ -16,12 +16,15 @@ export const ORCHESTRATION_COMMAND_SPECS: CommandSpec[] = [ }, { path: ['orchestration', 'run-use'], - summary: 'Bind this coordinator terminal to an existing Run', + summary: 'Claim coordinator authority for an existing Run', usage: 'orca orchestration run-use --id [--from ] [--takeover-legacy] [--retry-request ] [--json]', allowedFlags: [...GLOBAL_FLAGS, 'id', 'from', 'takeover-legacy', 'retry-request'], notes: [ - '--takeover-legacy must run in the live coordinator agent terminal it binds; it preserves existing worker assignments.' + 'Rebinding the same live coordinator preserves authority and Delivery. A different coordinator succeeds only when the owning host proves the prior coordinator exited.', + 'From the intended replacement agent, one run-use is the safe decision point: success preserves or transfers authority; rejection applies no effects and returns coordinatorStatus, claimantStatus, and exact recovery steps.', + 'A live or unverifiable owner returns consumer_fenced. Do not retry blindly, treat loss of contact as exit, infer an unowned Run from currentConsumer: false, or use --from to copy the owner handle.', + '--takeover-legacy explicitly replaces only the automatically adopted legacy Run coordinator. It is not a force override for ordinary Runs and must not be used while the legacy coordinator is actively coordinating.' ] }, { diff --git a/src/main/ipc/pty-controller-process-inventory.test.ts b/src/main/ipc/pty-controller-process-inventory.test.ts index e89d888978a5..c2325bac45b6 100644 --- a/src/main/ipc/pty-controller-process-inventory.test.ts +++ b/src/main/ipc/pty-controller-process-inventory.test.ts @@ -167,9 +167,21 @@ describe('aggregate PTY process inventory', () => { // Without a forwarded deadline an unanswered relay list runs to the SSH mux's own // 30s default, far past the runtime's 3s budget for the whole refresh. + expect(local.calls).toEqual([{ opts: { deadlineMs } }]) expect(remote.calls).toEqual([{ opts: { deadlineMs } }]) }) + it('forwards the caller deadline on a targeted local-provider list', async () => { + const local = createProvider([session('local-pty')]) + setLocalPtyProvider(local.provider) + const controller = captureController() + const deadlineMs = Date.now() + 1200 + + await controller.listProcesses(null, { deadlineMs }) + + expect(local.calls).toEqual([{ opts: { deadlineMs } }]) + }) + it('forwards the caller deadline on a targeted single-connection list', async () => { const local = createProvider([session('local-pty')]) const remote = createProvider([session('ssh:conn-a@@pty')]) diff --git a/src/main/ipc/pty/runtime/operations.ts b/src/main/ipc/pty/runtime/operations.ts index bf3ca74af25a..32d23f45c401 100644 --- a/src/main/ipc/pty/runtime/operations.ts +++ b/src/main/ipc/pty/runtime/operations.ts @@ -184,7 +184,7 @@ export async function listProcessesWithHostScopeFromRuntimeController( : LOCAL_EXECUTION_HOST_ID try { return { - processes: await (connectionId ? provider.listProcesses(opts) : provider.listProcesses()), + processes: await provider.listProcesses(opts), hostId } } catch (error) { @@ -209,7 +209,7 @@ export async function listProcessesFromRuntimeController( opts?: { deadlineMs?: number } ) { if (connectionId === null) { - return localProvider.listProcesses() + return localProvider.listProcesses(opts) } if (connectionId !== undefined) { try { diff --git a/src/main/runtime/orca-runtime-process-incarnation-liveness.test.ts b/src/main/runtime/orca-runtime-process-incarnation-liveness.test.ts index 3066367d7e83..94c3d27b4622 100644 --- a/src/main/runtime/orca-runtime-process-incarnation-liveness.test.ts +++ b/src/main/runtime/orca-runtime-process-incarnation-liveness.test.ts @@ -36,8 +36,12 @@ describe('terminal process incarnation liveness', () => { await expect( runtime.inspectTerminalProcessIncarnationLiveness('remote:ssh-1:pty-1:inc-old', SSH_SCOPE) ).resolves.toBe('exited') - expect(listProcesses).toHaveBeenNthCalledWith(1, 'ssh-1') - expect(listProcesses).toHaveBeenNthCalledWith(2, 'ssh-1') + expect(listProcesses).toHaveBeenNthCalledWith(1, 'ssh-1', { + deadlineMs: expect.any(Number) + }) + expect(listProcesses).toHaveBeenNthCalledWith(2, 'ssh-1', { + deadlineMs: expect.any(Number) + }) }) it('keeps missing or malformed identity and unavailable inventory unverifiable', async () => { @@ -61,6 +65,26 @@ describe('terminal process incarnation liveness', () => { ).resolves.toBe('unverifiable') }) + it('keeps a live legacy-provider process without incarnation identity unverifiable', async () => { + const ptyId = 'folder:workspace@@pty-1' + const runtime = runtimeWithInventory( + vi + .fn() + .mockResolvedValueOnce([{ id: ptyId, cwd: '', title: 'coordinator' }]) + .mockResolvedValueOnce([ + { id: ptyId, incarnationId: 'now-reported', cwd: '', title: 'coordinator' } + ]) + ) + + const inspect = () => + runtime.inspectTerminalProcessIncarnationLiveness( + `runtime-epoch:${ptyId}:7`, + JSON.stringify({ kind: 'local', hostId: 'local' }) + ) + await expect(inspect()).resolves.toBe('unverifiable') + await expect(inspect()).resolves.toBe('unverifiable') + }) + it('does not inspect an unproven host scope', async () => { const listProcesses = vi.fn().mockResolvedValue([]) const runtime = runtimeWithInventory(listProcesses) @@ -84,6 +108,8 @@ describe('terminal process incarnation liveness', () => { await expect( runtime.inspectTerminalProcessIncarnationLiveness('local-pty:inc-1', JSON.stringify(scope)) ).resolves.toBe('exited') - expect(listProcesses).toHaveBeenCalledWith(connectionId) + expect(listProcesses).toHaveBeenCalledWith(connectionId, { + deadlineMs: expect.any(Number) + }) }) }) diff --git a/src/main/runtime/orca-runtime.ts b/src/main/runtime/orca-runtime.ts index 4f494c2a3b57..a8bf3974db56 100644 --- a/src/main/runtime/orca-runtime.ts +++ b/src/main/runtime/orca-runtime.ts @@ -2449,6 +2449,7 @@ export type OrchestrationCompatibilityCallerAuthority = Readonly<{ terminalHandle: string processIncarnation: string launchTokenHash: string + terminalProvenance?: 'current_runtime' | 'restored' }> type RestoredOrchestrationAuthorityReceipt = Readonly<{ @@ -14540,7 +14541,8 @@ export class OrcaRuntimeService { terminal.processIncarnation, claimedPaneKey, terminalHandle, - launchTokenHash + launchTokenHash, + terminalProvenance ) } const attestation = this.attestAgentHookCompatibilityAuthorityFn?.({ @@ -14557,7 +14559,8 @@ export class OrcaRuntimeService { terminal.processIncarnation, attestation.paneKey, terminalHandle, - launchTokenHash + launchTokenHash, + terminalProvenance ) } @@ -14566,14 +14569,16 @@ export class OrcaRuntimeService { processIncarnation: string, paneKey: string, terminalHandle: string, - launchTokenHash: string + launchTokenHash: string, + terminalProvenance: 'current_runtime' | 'restored' ): OrchestrationCompatibilityCallerAuthority { return Object.freeze({ hostScope: Object.freeze({ ...terminal.hostScope }), paneKey, terminalHandle, processIncarnation, - launchTokenHash + launchTokenHash, + terminalProvenance }) } @@ -18144,8 +18149,11 @@ export class OrcaRuntimeService { if (!hostScope || !this.ptyController?.listProcesses) { return 'unverifiable' } + const deadlineMs = Date.now() + PTY_CONTROLLER_LIST_TIMEOUT_MS const listed = await withTimeoutResult( - this.ptyController.listProcesses(hostScope.kind === 'ssh' ? hostScope.targetId : null), + this.ptyController.listProcesses(hostScope.kind === 'ssh' ? hostScope.targetId : null, { + deadlineMs + }), PTY_CONTROLLER_LIST_TIMEOUT_MS ) if (!listed.ok) { diff --git a/src/main/runtime/orchestration-compatibility-authority.test.ts b/src/main/runtime/orchestration-compatibility-authority.test.ts index 082d6e698ae1..fefd9e71feec 100644 --- a/src/main/runtime/orchestration-compatibility-authority.test.ts +++ b/src/main/runtime/orchestration-compatibility-authority.test.ts @@ -70,7 +70,8 @@ describe('orchestration compatibility runtime authority', () => { paneKey: PANE_KEY, terminalHandle: 'term-1', processIncarnation: 'incarnation-1', - launchTokenHash: TOKEN_HASH + launchTokenHash: TOKEN_HASH, + terminalProvenance: 'current_runtime' }) expect(JSON.stringify(authority)).not.toContain(TOKEN) expect( diff --git a/src/main/runtime/orchestration-mailbox-detached-routing.test.ts b/src/main/runtime/orchestration-mailbox-detached-routing.test.ts index e1d484d3b14c..c675661b9976 100644 --- a/src/main/runtime/orchestration-mailbox-detached-routing.test.ts +++ b/src/main/runtime/orchestration-mailbox-detached-routing.test.ts @@ -146,7 +146,8 @@ describe('orchestration detached mailbox routing', () => { runId: currentRun!.id, coordinatorHandle: 'term_rebound_coordinator', coordinatorPaneKey: - '77777777-7777-4777-8777-777777777777:88888888-8888-4888-8888-888888888888' + '77777777-7777-4777-8777-777777777777:88888888-8888-4888-8888-888888888888', + authorityContinuity: true }) const checked = await checkBoundMailbox(harness.runtime) expect(checked).toMatchObject({ runId: workerRun.id, dispatchId: dispatch.id, count: 1 }) @@ -446,7 +447,8 @@ describe('orchestration detached mailbox routing', () => { runId: run.id, coordinatorHandle: 'term_new_coordinator', coordinatorPaneKey: - '55555555-5555-4555-8555-555555555555:66666666-6666-4666-8666-666666666666' + '55555555-5555-4555-8555-555555555555:66666666-6666-4666-8666-666666666666', + authorityContinuity: true }) expect(db.getMessageById(message.id)?.to_handle).toBe(TERMINAL_HANDLE) diff --git a/src/main/runtime/orchestration-mailbox-notification-consistency.test.ts b/src/main/runtime/orchestration-mailbox-notification-consistency.test.ts index bbcbc42e5648..9a05b4cfde3e 100644 --- a/src/main/runtime/orchestration-mailbox-notification-consistency.test.ts +++ b/src/main/runtime/orchestration-mailbox-notification-consistency.test.ts @@ -80,7 +80,8 @@ describe('orchestration notification mailbox consistency', () => { db.bindRun({ runId: runA.id, coordinatorHandle: SECOND_TERMINAL_HANDLE, - coordinatorPaneKey: SECOND_PANE_KEY + coordinatorPaneKey: SECOND_PANE_KEY, + authorityContinuity: true }) sqliteFor(db) .prepare('UPDATE messages SET to_handle = ? WHERE id = ?') @@ -359,7 +360,8 @@ describe('orchestration notification mailbox consistency', () => { runId: runA.id, coordinatorHandle: 'term_new_coordinator', coordinatorPaneKey: - '55555555-5555-4555-8555-555555555555:66666666-6666-4666-8666-666666666666' + '55555555-5555-4555-8555-555555555555:66666666-6666-4666-8666-666666666666', + authorityContinuity: true }) const runB = createBoundRun(db, 'Run B') const response = await new RpcDispatcher({ diff --git a/src/main/runtime/orchestration-mailbox-notification-test-harness.ts b/src/main/runtime/orchestration-mailbox-notification-test-harness.ts index 97426993a535..6a534bcedc41 100644 --- a/src/main/runtime/orchestration-mailbox-notification-test-harness.ts +++ b/src/main/runtime/orchestration-mailbox-notification-test-harness.ts @@ -22,6 +22,8 @@ export const SECOND_TERMINAL_HANDLE = 'term_mailbox_consistency_second' export const SECOND_LAUNCH_TOKEN = 'mailbox-consistency-second-launch' export const WORKTREE_ID = 'repo-mailbox::/tmp/mailbox' export const LAUNCH_TOKEN = 'mailbox-consistency-launch' +export const COORDINATOR_PROCESS_INCARNATION = `${PTY_ID}:mailbox-incarnation` +export const COORDINATOR_HOST_SCOPE = JSON.stringify({ kind: 'local', hostId: 'local' }) export const temporaryDirectories: string[] = [] export function createDatabase(prefix: string): OrchestrationDb { @@ -38,7 +40,9 @@ export function createBoundRun(db: OrchestrationDb, objective: string) { return db.createRun({ objective, coordinatorHandle: TERMINAL_HANDLE, - coordinatorPaneKey: PANE_KEY + coordinatorPaneKey: PANE_KEY, + coordinatorProcessIncarnation: COORDINATOR_PROCESS_INCARNATION, + coordinatorHostScope: COORDINATOR_HOST_SCOPE }) } diff --git a/src/main/runtime/orchestration-mailbox-routing-races.test.ts b/src/main/runtime/orchestration-mailbox-routing-races.test.ts index ff79fabc0655..59b61511460d 100644 --- a/src/main/runtime/orchestration-mailbox-routing-races.test.ts +++ b/src/main/runtime/orchestration-mailbox-routing-races.test.ts @@ -264,9 +264,10 @@ describe('orchestration mailbox routing races', () => { const harness = createRuntime(db) const run = db.createRun({ objective: 'Split Run handles', - coordinatorHandle: 'term_previous_coordinator', + coordinatorHandle: TERMINAL_HANDLE, coordinatorPaneKey: PANE_KEY }) + db.rememberRunCoordinatorHandle(run.id, 'term_previous_coordinator') const current = insertDirectRunMessage(db, run.id, 'Current handle') const previous = db.insertMessage({ from: 'term_worker', diff --git a/src/main/runtime/orchestration-message-delivery-identity.test.ts b/src/main/runtime/orchestration-message-delivery-identity.test.ts index 92aa7e907484..d9d9113a274a 100644 --- a/src/main/runtime/orchestration-message-delivery-identity.test.ts +++ b/src/main/runtime/orchestration-message-delivery-identity.test.ts @@ -25,6 +25,8 @@ const TERMINAL_HANDLE = 'term_sta_4325' const REMINTED_TERMINAL_HANDLE = 'term_sta_4325_reminted' const WORKTREE_ID = 'repo-sta-4325::/tmp/sta-4325' const LAUNCH_TOKEN = 'sta-4325-launch' +const COORDINATOR_PROCESS_INCARNATION = `${PTY_ID}:sta-4325-incarnation` +const COORDINATOR_HOST_SCOPE = JSON.stringify({ kind: 'local', hostId: 'local' }) const temporaryDirectories: string[] = [] const CLI_PATH = join(process.cwd(), 'out', 'cli', 'index.js') const itIfCliBuilt = existsSync(CLI_PATH) ? it : it.skip @@ -148,7 +150,8 @@ async function runBuiltCli( ...process.env, ORCA_USER_DATA_PATH: userDataPath, ORCA_TERMINAL_HANDLE: TERMINAL_HANDLE, - ORCA_PANE_KEY: PANE_KEY + ORCA_PANE_KEY: PANE_KEY, + ORCA_AGENT_LAUNCH_TOKEN: LAUNCH_TOKEN }, stdio: ['ignore', 'pipe', 'pipe'] }) @@ -268,7 +271,9 @@ describe('STA-4325 message and delivery identity', () => { const run = fixture.db.createRun({ objective: 'STA-4325 restart', coordinatorHandle: TERMINAL_HANDLE, - coordinatorPaneKey: PANE_KEY + coordinatorPaneKey: PANE_KEY, + coordinatorProcessIncarnation: COORDINATOR_PROCESS_INCARNATION, + coordinatorHostScope: COORDINATOR_HOST_SCOPE }) const status = fixture.db.insertMessage({ from: 'term_worker', @@ -389,12 +394,17 @@ describe('STA-4325 message and delivery identity', () => { const run = fixture.db.createRun({ objective: 'Late old-handle arrival', coordinatorHandle: TERMINAL_HANDLE, - coordinatorPaneKey: PANE_KEY + coordinatorPaneKey: PANE_KEY, + coordinatorProcessIncarnation: COORDINATOR_PROCESS_INCARNATION, + coordinatorHostScope: COORDINATOR_HOST_SCOPE }) fixture.db.bindRun({ runId: run.id, coordinatorHandle: REMINTED_TERMINAL_HANDLE, - coordinatorPaneKey: PANE_KEY + coordinatorPaneKey: PANE_KEY, + coordinatorProcessIncarnation: COORDINATOR_PROCESS_INCARNATION, + coordinatorHostScope: COORDINATOR_HOST_SCOPE, + authorityContinuity: true }) const done = fixture.db.insertMessage({ from: 'term_worker', @@ -435,11 +445,7 @@ describe('STA-4325 message and delivery identity', () => { const userDataPath = mkdtempSync(join(tmpdir(), 'orca-sta-4325-cli-')) temporaryDirectories.push(userDataPath) const db = new OrchestrationDb(join(userDataPath, 'orchestration.db')) - const runtime = new OrcaRuntimeService() - runtime.setOrchestrationDb(db) - vi.spyOn(runtime, 'getTerminalPaneKey').mockImplementation((handle) => - handle === TERMINAL_HANDLE ? PANE_KEY : null - ) + const { runtime } = createRuntime(db) const run = db.createRun({ objective: 'STA-4325 built CLI', coordinatorHandle: TERMINAL_HANDLE, diff --git a/src/main/runtime/orchestration/db/contract-constants.ts b/src/main/runtime/orchestration/db/contract-constants.ts index 56390138f0a7..cf29b2cba604 100644 --- a/src/main/runtime/orchestration/db/contract-constants.ts +++ b/src/main/runtime/orchestration/db/contract-constants.ts @@ -6,5 +6,5 @@ export const LEGACY_RUN_ID = ORCHESTRATION_LEGACY_RUN_ID export const LEGACY_CONTRACT_VERSION = 0 export const CURRENT_CONTRACT_VERSION = ORCHESTRATION_CONTRACT_VERSION -// Schema versions: v2 'heartbeat'+last_heartbeat_at, v3 delivered_at, v4 task-creator terminal, v5 task_title/display_name, v6 pane identity, v7 lightweight Runs, v8 crash-safe Run deliveries, v9 durable question threads, v10 Dispatch capabilities, v11 durable mutation receipts, v12 composed worker state, v18 post-v6 version-skew repair, v19 adopted legacy Runs and compatibility receipts, v20 legacy question backfill, v21 legacy scheduler-loss provenance, v22 dispatch assignee lookup, v23 worker terminal resource ownership, v24 creator-incarnation authority, v25 active Dispatch handle lookup, v26 indexed mutation receipt capacity, v27 durable federation acknowledgments, v28 durable local mutation caller identity. -export const SCHEMA_VERSION = 30 +// Schema versions: v2 'heartbeat'+last_heartbeat_at, v3 delivered_at, v4 task-creator terminal, v5 task_title/display_name, v6 pane identity, v7 lightweight Runs, v8 crash-safe Run deliveries, v9 durable question threads, v10 Dispatch capabilities, v11 durable mutation receipts, v12 composed worker state, v18 post-v6 version-skew repair, v19 adopted legacy Runs and compatibility receipts, v20 legacy question backfill, v21 legacy scheduler-loss provenance, v22 dispatch assignee lookup, v23 worker terminal resource ownership, v24 creator-incarnation authority, v25 active Dispatch handle lookup, v26 indexed mutation receipt capacity, v27 durable federation acknowledgments, v28 durable local mutation caller identity, v31 Run coordinator process authority. +export const SCHEMA_VERSION = 31 diff --git a/src/main/runtime/orchestration/db/runs/run-binding.ts b/src/main/runtime/orchestration/db/runs/run-binding.ts index a4110dd589f9..58bf74657e65 100644 --- a/src/main/runtime/orchestration/db/runs/run-binding.ts +++ b/src/main/runtime/orchestration/db/runs/run-binding.ts @@ -1,15 +1,28 @@ import type { RunRow } from '../../types' import { OrchestrationError } from '../../orchestration-error' +import { isCurrentRunCoordinator } from '../../run-coordinator-authority' import { LEGACY_CONTRACT_VERSION } from '../contract-constants' import { isEquivalentPaneKey } from '../pane-key-match' import type { OrchestrationDb } from '../orchestration-db' +export type RunCoordinatorObservation = { + coordinatorHandle: string | null + coordinatorPaneKey: string | null + coordinatorProcessIncarnation: string | null + coordinatorHostScope: string | null + status: 'live' | 'unverifiable' | 'exited' +} + export function bindRun( this: OrchestrationDb, params: { runId: string coordinatorHandle: string coordinatorPaneKey: string + coordinatorProcessIncarnation?: string | null + coordinatorHostScope?: string | null + authorityContinuity?: boolean + incumbentObservation?: RunCoordinatorObservation takeoverLegacy?: boolean legacyCoordinatorAuthority?: { runId: string @@ -30,8 +43,29 @@ export function bindRun( const sameBinding = run.coordinator_pane_key !== null && isEquivalentPaneKey(run.coordinator_pane_key, params.coordinatorPaneKey) + const sameAuthority = + params.authorityContinuity === true || + isCurrentRunCoordinator(run, { + handle: params.coordinatorHandle, + paneKey: params.coordinatorPaneKey, + processIncarnation: params.coordinatorProcessIncarnation ?? null, + hostScope: params.coordinatorHostScope ?? null + }) const adoption = this.getLegacyAdoption() const adoptedRun = adoption?.adopted_run_id === params.runId + const observation = params.incumbentObservation + const observationMatches = + observation?.coordinatorHandle === run.coordinator_handle && + observation.coordinatorPaneKey === run.coordinator_pane_key && + observation.coordinatorProcessIncarnation === run.coordinator_process_incarnation && + observation.coordinatorHostScope === run.coordinator_host_scope + const hasIncumbentAuthority = Boolean( + run.coordinator_handle || + run.coordinator_pane_key || + run.coordinator_process_incarnation || + run.coordinator_host_scope + ) + const requiresAuthorityProof = hasIncumbentAuthority const legacyAuthority = params.legacyCoordinatorAuthority const legacyPrincipalId = legacyAuthority?.principalId const legacyPrincipal = legacyPrincipalId @@ -83,6 +117,10 @@ export function bindRun( run.coordinator_handle === params.coordinatorHandle && coordinatorPrincipal?.status !== 'committed' ) + const appliesLegacyTakeover = Boolean( + params.takeoverLegacy && + (coordinatorPrincipal?.status === 'committed' || !hasIncumbentAuthority) + ) const replacesLegacyCoordinator = Boolean( adoptedRun && !provenLegacyBinding && @@ -99,7 +137,7 @@ export function bindRun( } // Why: only LIVE legacy work needs the flag — settled work has no competing authority left, and // fencing it would strand the recovered graph behind an attestation the caller may not have. - if (activeLegacyAssignment && !sameBinding && !provenLegacyBinding && !params.takeoverLegacy) { + if (activeLegacyAssignment && !sameBinding && !provenLegacyBinding && !appliesLegacyTakeover) { throw new OrchestrationError( 'consumer_fenced', 'This adopted Run still has live legacy work. Its attested coordinator may rebind it, or a current coordinator may explicitly use run-use --takeover-legacy.', @@ -109,7 +147,54 @@ export function bindRun( } ) } - this.unbindOtherRunsForPane(params.coordinatorPaneKey, params.runId) + if ( + requiresAuthorityProof && + !sameAuthority && + !provenLegacyBinding && + !appliesLegacyTakeover && + (!observationMatches || observation.status !== 'exited') + ) { + const coordinatorStatus = observationMatches ? observation.status : 'unverifiable' + const inspectCommand = `orchestration run-show --id ${params.runId} --json` + const retryCommand = `orchestration run-use --id ${params.runId} --json` + const nextSteps = + coordinatorStatus === 'live' + ? [ + 'Continue from the owning coordinator terminal; this caller has read-only inspection authority.', + `To intentionally transfer authority, stop or exit that coordinator process, then run ${retryCommand} with the same Orca CLI executable from the replacement.`, + `Inspect current authority by running ${inspectCommand} with the same Orca CLI executable; binding.currentConsumer is true only for the owner.`, + 'Do not retry while coordinatorStatus is live. No force-steal exists for an ordinary Run.' + ] + : [ + 'Restore connectivity to the owning host. Loss of contact is not evidence of exit.', + `Inspect current authority by running ${inspectCommand} with the same Orca CLI executable; binding.currentConsumer is true only for the owner.`, + `Run ${retryCommand} with that executable only after the owning host proves the incumbent exited.`, + 'If the incumbent is live, continue from its coordinator terminal; no force-steal exists for an ordinary Run.' + ] + throw new OrchestrationError( + 'consumer_fenced', + coordinatorStatus === 'live' + ? 'This Run is owned by another live coordinator. No effects were applied.' + : 'This Run coordinator could not be proven exited. Retry from the owning coordinator terminal after connectivity is restored. No effects were applied.', + { + effectsApplied: false, + coordinatorStatus, + inspectCommandArgs: ['orchestration', 'run-show', '--id', params.runId, '--json'], + retryCommandArgs: ['orchestration', 'run-use', '--id', params.runId, '--json'], + nextSteps + } + ) + } + this.unbindOtherRunsForPane( + params.coordinatorPaneKey, + { + handle: params.coordinatorHandle, + paneKey: params.coordinatorPaneKey, + processIncarnation: params.coordinatorProcessIncarnation ?? null, + hostScope: params.coordinatorHostScope ?? null + }, + params.runId + ) for (const handle of new Set( [run.coordinator_handle, params.coordinatorHandle].filter((value): value is string => Boolean(value) @@ -118,32 +203,45 @@ export function bindRun( this.rememberRunCoordinatorHandle(params.runId, handle) this.routeAllUnreadDirectMessagesToRunMailbox(params.runId, handle) } - if ( - (params.takeoverLegacy && !takeoverAlreadyApplied) || + const bindingMetadataChanged = !sameBinding || - run.coordinator_handle !== params.coordinatorHandle - ) { - if (adoptedRun && (params.takeoverLegacy || !activeLegacyAssignment)) { + run.coordinator_handle !== params.coordinatorHandle || + run.coordinator_process_incarnation !== (params.coordinatorProcessIncarnation ?? null) || + run.coordinator_host_scope !== (params.coordinatorHostScope ?? null) + if ((appliesLegacyTakeover && !takeoverAlreadyApplied) || bindingMetadataChanged) { + if (adoptedRun && (appliesLegacyTakeover || !activeLegacyAssignment)) { if ( coordinatorPrincipal?.status === 'committed' && - (params.takeoverLegacy || + (appliesLegacyTakeover || coordinatorPrincipal.terminal_handle !== params.coordinatorHandle || !isEquivalentPaneKey(coordinatorPrincipal.pane_key, params.coordinatorPaneKey)) ) { this.setLegacyCompatibilityPrincipalStatus(coordinatorPrincipal.id, 'revoked') } } + const changesAuthority = !sameAuthority || appliesLegacyTakeover this.db .prepare( `UPDATE runs SET coordinator_handle = ?, coordinator_pane_key = ?, - consumer_generation = consumer_generation + 1, + coordinator_process_incarnation = ?, coordinator_host_scope = ?, + coordinator_authority_revision = coordinator_authority_revision + 1, + consumer_generation = consumer_generation + ?, updated_at = datetime('now') WHERE id = ?` ) - .run(params.coordinatorHandle, params.coordinatorPaneKey, params.runId) - this.fenceOutstandingDelivery(params.runId) - if (params.takeoverLegacy || replacesLegacyCoordinator) { + .run( + params.coordinatorHandle, + params.coordinatorPaneKey, + params.coordinatorProcessIncarnation ?? null, + params.coordinatorHostScope ?? null, + changesAuthority ? 1 : 0, + params.runId + ) + if (changesAuthority) { + this.fenceOutstandingDelivery(params.runId) + } + if (appliesLegacyTakeover || replacesLegacyCoordinator) { this.promoteLegacyCoordinatorMailForTakeover(params.runId, retainedCoordinatorHandle) } } diff --git a/src/main/runtime/orchestration/db/runs/run-create.ts b/src/main/runtime/orchestration/db/runs/run-create.ts index 3f99e0189362..21bfb555c2df 100644 --- a/src/main/runtime/orchestration/db/runs/run-create.ts +++ b/src/main/runtime/orchestration/db/runs/run-create.ts @@ -10,20 +10,35 @@ export function createRun( objective: string coordinatorHandle: string coordinatorPaneKey: string + coordinatorProcessIncarnation?: string | null + coordinatorHostScope?: string | null } ): RunRow { const id = generateId('run') this.db.exec('BEGIN IMMEDIATE') try { - this.unbindOtherRunsForPane(params.coordinatorPaneKey) + this.unbindOtherRunsForPane(params.coordinatorPaneKey, { + handle: params.coordinatorHandle, + paneKey: params.coordinatorPaneKey, + processIncarnation: params.coordinatorProcessIncarnation ?? null, + hostScope: params.coordinatorHostScope ?? null + }) this.db .prepare( `INSERT INTO runs ( id, objective, coordinator_handle, coordinator_pane_key, - consumer_generation, legacy - ) VALUES (?, ?, ?, ?, 1, 0)` + coordinator_process_incarnation, coordinator_host_scope, + coordinator_authority_revision, consumer_generation, legacy + ) VALUES (?, ?, ?, ?, ?, ?, 0, 1, 0)` + ) + .run( + id, + params.objective, + params.coordinatorHandle, + params.coordinatorPaneKey, + params.coordinatorProcessIncarnation ?? null, + params.coordinatorHostScope ?? null ) - .run(id, params.objective, params.coordinatorHandle, params.coordinatorPaneKey) this.rememberRunCoordinatorHandle(id, params.coordinatorHandle) this.db.exec('COMMIT') } catch (error) { diff --git a/src/main/runtime/orchestration/db/runs/run-lookup.ts b/src/main/runtime/orchestration/db/runs/run-lookup.ts index 061a7b39497c..ce3ed5110aa9 100644 --- a/src/main/runtime/orchestration/db/runs/run-lookup.ts +++ b/src/main/runtime/orchestration/db/runs/run-lookup.ts @@ -1,4 +1,9 @@ import type { RunRow } from '../../types' +import { OrchestrationError } from '../../orchestration-error' +import { + isCurrentRunCoordinator, + type RunCoordinatorIdentity +} from '../../run-coordinator-authority' import { ORCHESTRATION_RUN_PAGE_LIMIT } from '../../../../../shared/orchestration-run-pagination' import { isEquivalentPaneKey, @@ -124,10 +129,27 @@ export function getRunRaw(this: OrchestrationDb, id: string): RunRow | undefined export function unbindOtherRunsForPane( this: OrchestrationDb, paneKey: string, + identity: RunCoordinatorIdentity, exceptRunId?: string ): void { for (const run of this.runsBoundToPane(paneKey)) { if (run.id !== exceptRunId) { + if (!isCurrentRunCoordinator(run, identity)) { + throw new OrchestrationError( + 'consumer_fenced', + `Run ${run.id} is owned by another coordinator process in this pane. No effects were applied.`, + { + effectsApplied: false, + inspectCommandArgs: ['orchestration', 'run-show', '--id', run.id, '--json'], + retryCommandArgs: ['orchestration', 'run-use', '--id', run.id, '--json'], + nextSteps: [ + `Inspect current authority by running orchestration run-show --id ${run.id} --json with the same Orca CLI executable.`, + `To switch Runs from this replacement process, first run orchestration run-use --id ${run.id} --json with that executable after its owning host proves the incumbent exited.`, + 'Do not retry the create or bind command unchanged while another process owns the pane.' + ] + } + ) + } if (run.coordinator_handle) { this.routeAllUnreadDirectMessagesToRunMailbox(run.id, run.coordinator_handle) } @@ -135,6 +157,8 @@ export function unbindOtherRunsForPane( .prepare( `UPDATE runs SET coordinator_handle = NULL, coordinator_pane_key = NULL, + coordinator_process_incarnation = NULL, coordinator_host_scope = NULL, + coordinator_authority_revision = coordinator_authority_revision + 1, consumer_generation = consumer_generation + 1, updated_at = datetime('now') WHERE id = ?` diff --git a/src/main/runtime/orchestration/db/schema/create-core-tables-sql.ts b/src/main/runtime/orchestration/db/schema/create-core-tables-sql.ts index 1b3172edf31c..5cac9c35c8fe 100644 --- a/src/main/runtime/orchestration/db/schema/create-core-tables-sql.ts +++ b/src/main/runtime/orchestration/db/schema/create-core-tables-sql.ts @@ -8,6 +8,9 @@ CREATE TABLE IF NOT EXISTS runs ( home_database TEXT NOT NULL DEFAULT 'this_database', coordinator_handle TEXT, coordinator_pane_key TEXT, + coordinator_process_incarnation TEXT, + coordinator_host_scope TEXT, + coordinator_authority_revision INTEGER NOT NULL DEFAULT 0, consumer_generation INTEGER NOT NULL DEFAULT 0, legacy INTEGER NOT NULL DEFAULT 0, created_at TEXT NOT NULL DEFAULT (datetime('now')), @@ -75,6 +78,20 @@ BEGIN DELETE FROM run_coordinator_handles WHERE run_id = OLD.id; END; +CREATE TRIGGER IF NOT EXISTS trg_runs_clear_stale_coordinator_authority +AFTER UPDATE OF coordinator_handle, coordinator_pane_key ON runs +WHEN NEW.coordinator_authority_revision = OLD.coordinator_authority_revision + AND NOT ( + NEW.coordinator_handle IS OLD.coordinator_handle + AND NEW.coordinator_pane_key IS OLD.coordinator_pane_key + ) +BEGIN + UPDATE runs + SET coordinator_process_incarnation = NULL, coordinator_host_scope = NULL, + coordinator_authority_revision = -1 + WHERE id = NEW.id; +END; + CREATE TABLE IF NOT EXISTS deliveries ( id TEXT PRIMARY KEY, run_id TEXT NOT NULL, diff --git a/src/main/runtime/orchestration/db/schema/migrate-v31.ts b/src/main/runtime/orchestration/db/schema/migrate-v31.ts new file mode 100644 index 000000000000..c3ed5d4104d4 --- /dev/null +++ b/src/main/runtime/orchestration/db/schema/migrate-v31.ts @@ -0,0 +1,33 @@ +import type { OrchestrationDb } from '../orchestration-db' + +export function applySchemaMigrationV31(this: OrchestrationDb, current: number): void { + if (current >= 31) { + return + } + if (!this.hasColumn('runs', 'coordinator_process_incarnation')) { + this.db.exec('ALTER TABLE runs ADD COLUMN coordinator_process_incarnation TEXT') + } + if (!this.hasColumn('runs', 'coordinator_host_scope')) { + this.db.exec('ALTER TABLE runs ADD COLUMN coordinator_host_scope TEXT') + } + if (!this.hasColumn('runs', 'coordinator_authority_revision')) { + this.db.exec( + 'ALTER TABLE runs ADD COLUMN coordinator_authority_revision INTEGER NOT NULL DEFAULT -1' + ) + } + this.db.exec(` + CREATE TRIGGER IF NOT EXISTS trg_runs_clear_stale_coordinator_authority + AFTER UPDATE OF coordinator_handle, coordinator_pane_key ON runs + WHEN NEW.coordinator_authority_revision = OLD.coordinator_authority_revision + AND NOT ( + NEW.coordinator_handle IS OLD.coordinator_handle + AND NEW.coordinator_pane_key IS OLD.coordinator_pane_key + ) + BEGIN + UPDATE runs + SET coordinator_process_incarnation = NULL, coordinator_host_scope = NULL, + coordinator_authority_revision = -1 + WHERE id = NEW.id; + END; + `) +} diff --git a/src/main/runtime/orchestration/db/schema/migrate.ts b/src/main/runtime/orchestration/db/schema/migrate.ts index b29debf6aa13..71d6923b465f 100644 --- a/src/main/runtime/orchestration/db/schema/migrate.ts +++ b/src/main/runtime/orchestration/db/schema/migrate.ts @@ -3,6 +3,7 @@ import { SCHEMA_VERSION } from '../contract-constants' import type { OrchestrationDb } from '../orchestration-db' import { applySchemaMigrationsV13ToV30 } from './migrate-v13-v30' import { applySchemaMigrationsV2ToV12 } from './migrate-v2-v12' +import { applySchemaMigrationV31 } from './migrate-v31' // Why: CREATE TABLE IF NOT EXISTS won't alter existing DBs; migrate in a txn that bumps user_version only on success (atomic all-or-nothing). export function migrate(this: OrchestrationDb): void { @@ -16,6 +17,7 @@ export function migrate(this: OrchestrationDb): void { try { applySchemaMigrationsV2ToV12.call(this, current) applySchemaMigrationsV13ToV30.call(this, current) + applySchemaMigrationV31.call(this, current) this.db.pragma(`user_version = ${SCHEMA_VERSION}`) this.db.exec('COMMIT') } catch (err) { diff --git a/src/main/runtime/orchestration/orchestration-adopted-run-binding.test.ts b/src/main/runtime/orchestration/orchestration-adopted-run-binding.test.ts index 800a7511451f..3e8910ab4219 100644 --- a/src/main/runtime/orchestration/orchestration-adopted-run-binding.test.ts +++ b/src/main/runtime/orchestration/orchestration-adopted-run-binding.test.ts @@ -184,8 +184,8 @@ describe('adopted Run binding without --takeover-legacy', () => { expect(db.getLegacyCompatibilityPrincipal(principalId)?.status).toBe('revoked') }) - // Why: the acknowledgement is per claimant, not one-shot — a prior takeover does not license the next. - it('keeps fencing new claimants while legacy work stays live', () => { + // Why: the flag authorizes only the legacy-to-current cutover, never a second current coordinator. + it('keeps fencing flagged claimants after legacy takeover', () => { const { db, adoptedRunId } = createAdoptedFixture({ settleWork: false }) db.bindRun({ runId: adoptedRunId, @@ -198,24 +198,44 @@ describe('adopted Run binding without --takeover-legacy', () => { db.bindRun({ runId: adoptedRunId, coordinatorHandle: 'term_second_coord', - coordinatorPaneKey: 'tab_second:22222222-2222-4222-8222-222222222222' + coordinatorPaneKey: 'tab_second:22222222-2222-4222-8222-222222222222', + takeoverLegacy: true }) ).toThrowError(expect.objectContaining({ code: 'consumer_fenced' })) }) - it('lets a later coordinator claim the settled Run once the first one has', () => { + it('requires proven exit before a later coordinator replaces a settled Run claimant', () => { const { db, adoptedRunId } = createAdoptedFixture({ settleWork: true }) db.bindRun({ runId: adoptedRunId, coordinatorHandle: CURRENT_COORDINATOR_HANDLE, - coordinatorPaneKey: CURRENT_COORDINATOR_PANE + coordinatorPaneKey: CURRENT_COORDINATOR_PANE, + coordinatorProcessIncarnation: 'pty-current:incarnation-1', + coordinatorHostScope: JSON.stringify({ kind: 'local', hostId: 'local' }) }) + const observation = { + coordinatorHandle: CURRENT_COORDINATOR_HANDLE, + coordinatorPaneKey: CURRENT_COORDINATOR_PANE, + coordinatorProcessIncarnation: 'pty-current:incarnation-1', + coordinatorHostScope: JSON.stringify({ kind: 'local', hostId: 'local' }), + status: 'live' as const + } + expect(() => + db.bindRun({ + runId: adoptedRunId, + coordinatorHandle: 'term_second_coord', + coordinatorPaneKey: 'tab_second:22222222-2222-4222-8222-222222222222', + incumbentObservation: observation + }) + ).toThrowError(expect.objectContaining({ code: 'consumer_fenced' })) + expect( db.bindRun({ runId: adoptedRunId, coordinatorHandle: 'term_second_coord', - coordinatorPaneKey: 'tab_second:22222222-2222-4222-8222-222222222222' + coordinatorPaneKey: 'tab_second:22222222-2222-4222-8222-222222222222', + incumbentObservation: { ...observation, status: 'exited' } }) ).toMatchObject({ coordinator_handle: 'term_second_coord', consumer_generation: 2 }) }) @@ -283,7 +303,9 @@ describe('pane-bound Run lookup', () => { const shared = db.createRun({ objective: 'first', coordinatorHandle: 'term_d', - coordinatorPaneKey: 'tab_one:99999999-9999-4999-8999-999999999999' + coordinatorPaneKey: 'tab_one:99999999-9999-4999-8999-999999999999', + coordinatorProcessIncarnation: 'pty-d:incarnation-1', + coordinatorHostScope: JSON.stringify({ kind: 'local', hostId: 'local' }) }) const untouched = db.createRun({ objective: 'other pane', @@ -294,12 +316,16 @@ describe('pane-bound Run lookup', () => { const rebound = db.createRun({ objective: 'second', coordinatorHandle: 'term_d', - coordinatorPaneKey: 'tab_reminted:99999999-9999-4999-8999-999999999999' + coordinatorPaneKey: 'tab_reminted:99999999-9999-4999-8999-999999999999', + coordinatorProcessIncarnation: 'pty-d:incarnation-1', + coordinatorHostScope: JSON.stringify({ kind: 'local', hostId: 'local' }) }) // createRun binds at generation 1; the unbind fences it to 2. expect(db.getRun(shared.id)).toMatchObject({ coordinator_pane_key: null, + coordinator_process_incarnation: null, + coordinator_host_scope: null, consumer_generation: 2 }) expect(db.getRun(untouched.id)).toMatchObject({ diff --git a/src/main/runtime/orchestration/orchestration-run-delivery-db.test.ts b/src/main/runtime/orchestration/orchestration-run-delivery-db.test.ts index fdaa5e92a801..9595bee2e562 100644 --- a/src/main/runtime/orchestration/orchestration-run-delivery-db.test.ts +++ b/src/main/runtime/orchestration/orchestration-run-delivery-db.test.ts @@ -142,7 +142,14 @@ describe('OrchestrationDb Run state', () => { const rebound = d.bindRun({ runId: run.id, coordinatorHandle: 'term_new', - coordinatorPaneKey: 'tab_new:22222222-2222-4222-9222-222222222222' + coordinatorPaneKey: 'tab_new:22222222-2222-4222-9222-222222222222', + incumbentObservation: { + coordinatorHandle: 'term_coord', + coordinatorPaneKey: 'tab_coord:11111111-1111-4111-8111-111111111111', + coordinatorProcessIncarnation: null, + coordinatorHostScope: null, + status: 'exited' + } })! let fencedError: unknown @@ -164,6 +171,38 @@ describe('OrchestrationDb Run state', () => { expect(replacement?.messages.map((message) => message.subject)).toEqual(['one']) }) + it('preserves an outstanding batch across a same-process handle remint', () => { + const d = createDb() + const run = d.createRun({ + objective: 'Mailbox continuity', + coordinatorHandle: 'term_old', + coordinatorPaneKey: 'tab_old:11111111-1111-4111-8111-111111111111', + coordinatorProcessIncarnation: 'pty:incarnation-1', + coordinatorHostScope: JSON.stringify({ kind: 'local', hostId: 'local' }) + }) + d.insertMessage({ from: 'a', to: `run:${run.id}`, subject: 'one', runId: run.id }) + const delivery = d.getOrCreateRunDelivery({ + runId: run.id, + consumerGeneration: run.consumer_generation + })! + + const rebound = d.bindRun({ + runId: run.id, + coordinatorHandle: 'term_reminted', + coordinatorPaneKey: 'tab_reminted:22222222-2222-4222-9222-222222222222', + coordinatorProcessIncarnation: 'pty:incarnation-1', + coordinatorHostScope: JSON.stringify({ kind: 'local', hostId: 'local' }) + })! + + expect(rebound.consumer_generation).toBe(run.consumer_generation) + expect( + d.getOrCreateRunDelivery({ + runId: run.id, + consumerGeneration: run.consumer_generation + })?.delivery.id + ).toBe(delivery.delivery.id) + }) + it('does not move a mismatched Run through another Run Dispatch mailbox', () => { const d = createDb() const runA = createBoundRun(d) @@ -228,7 +267,9 @@ describe('OrchestrationDb Run state', () => { const first = d.createRun({ objective: 'First objective', coordinatorHandle: 'term_first', - coordinatorPaneKey: 'tab_a:11111111-1111-4111-8111-111111111111' + coordinatorPaneKey: 'tab_a:11111111-1111-4111-8111-111111111111', + coordinatorProcessIncarnation: 'pty:incarnation-1', + coordinatorHostScope: JSON.stringify({ kind: 'local', hostId: 'local' }) }) expect(first).toMatchObject({ consumer_generation: 1, legacy: 0 }) expect(d.getCurrentRunForPane('tab_reminted:11111111-1111-4111-8111-111111111111')?.id).toBe( @@ -238,7 +279,9 @@ describe('OrchestrationDb Run state', () => { const second = d.createRun({ objective: 'Second objective', coordinatorHandle: 'term_second', - coordinatorPaneKey: 'tab_b:11111111-1111-4111-8111-111111111111' + coordinatorPaneKey: 'tab_b:11111111-1111-4111-8111-111111111111', + coordinatorProcessIncarnation: 'pty:incarnation-1', + coordinatorHostScope: JSON.stringify({ kind: 'local', hostId: 'local' }) }) expect(d.getRun(first.id)).toMatchObject({ coordinator_handle: null, @@ -250,7 +293,7 @@ describe('OrchestrationDb Run state', () => { ) }) - it('rebinds a Run by incrementing its consumer generation', () => { + it('rebinds a Run after proven coordinator exit by incrementing its consumer generation', () => { const d = createDb() const run = d.createRun({ objective: 'Move coordinator', @@ -262,7 +305,14 @@ describe('OrchestrationDb Run state', () => { d.bindRun({ runId: run.id, coordinatorHandle: 'term_new', - coordinatorPaneKey: 'tab_new:22222222-2222-4222-9222-222222222222' + coordinatorPaneKey: 'tab_new:22222222-2222-4222-9222-222222222222', + incumbentObservation: { + coordinatorHandle: 'term_old', + coordinatorPaneKey: 'tab_old:11111111-1111-4111-8111-111111111111', + coordinatorProcessIncarnation: null, + coordinatorHostScope: null, + status: 'exited' + } }) ).toMatchObject({ coordinator_handle: 'term_new', @@ -278,6 +328,49 @@ describe('OrchestrationDb Run state', () => { ).toBeUndefined() }) + it('fences a second takeover that races using a stale exited-owner observation', () => { + const d = createDb() + const oldPane = 'tab_old:11111111-1111-4111-8111-111111111111' + const run = d.createRun({ + objective: 'Single winner', + coordinatorHandle: 'term_old', + coordinatorPaneKey: oldPane, + coordinatorProcessIncarnation: 'pty-old:incarnation-1', + coordinatorHostScope: JSON.stringify({ kind: 'ssh', targetId: 'target-1' }) + }) + const observation = { + coordinatorHandle: 'term_old', + coordinatorPaneKey: oldPane, + coordinatorProcessIncarnation: 'pty-old:incarnation-1', + coordinatorHostScope: JSON.stringify({ kind: 'ssh', targetId: 'target-1' }), + status: 'exited' as const + } + + d.bindRun({ + runId: run.id, + coordinatorHandle: 'term_winner', + coordinatorPaneKey: 'tab_winner:22222222-2222-4222-9222-222222222222', + coordinatorProcessIncarnation: 'pty-winner:incarnation-1', + coordinatorHostScope: JSON.stringify({ kind: 'local', hostId: 'local' }), + incumbentObservation: observation + }) + + expect(() => + d.bindRun({ + runId: run.id, + coordinatorHandle: 'term_loser', + coordinatorPaneKey: 'tab_loser:33333333-3333-4333-8333-333333333333', + coordinatorProcessIncarnation: 'pty-loser:incarnation-1', + coordinatorHostScope: JSON.stringify({ kind: 'wsl', hostId: 'local', distro: 'Ubuntu' }), + incumbentObservation: observation + }) + ).toThrow(expect.objectContaining({ code: 'consumer_fenced' })) + expect(d.getRun(run.id)).toMatchObject({ + coordinator_handle: 'term_winner', + consumer_generation: run.consumer_generation + 1 + }) + }) + it('associates task, dispatch, message, and gate rows with the selected Run', () => { const d = createDb() const run = d.createRun({ diff --git a/src/main/runtime/orchestration/orchestration-schema-version-skew.ts b/src/main/runtime/orchestration/orchestration-schema-version-skew.ts index 95ecbe6b0346..19e49f7a296a 100644 --- a/src/main/runtime/orchestration/orchestration-schema-version-skew.ts +++ b/src/main/runtime/orchestration/orchestration-schema-version-skew.ts @@ -28,7 +28,10 @@ const POST_V6_COLUMNS = [ const VERSIONED_POST_V6_COLUMNS = [ { version: 27, table: 'federated_dispatches', column: 'to_home_acknowledged_sequence' }, { version: 30, table: 'dispatch_contexts', column: 'depth' }, - { version: 30, table: 'remote_dispatch_attachments', column: 'depth' } + { version: 30, table: 'remote_dispatch_attachments', column: 'depth' }, + { version: 31, table: 'runs', column: 'coordinator_process_incarnation' }, + { version: 31, table: 'runs', column: 'coordinator_host_scope' }, + { version: 31, table: 'runs', column: 'coordinator_authority_revision' } ] as const const POST_V6_INDEXES = [ diff --git a/src/main/runtime/orchestration/run-coordinator-authority-migration.test.ts b/src/main/runtime/orchestration/run-coordinator-authority-migration.test.ts new file mode 100644 index 000000000000..122328545ff0 --- /dev/null +++ b/src/main/runtime/orchestration/run-coordinator-authority-migration.test.ts @@ -0,0 +1,121 @@ +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import Database from '../../sqlite/sync-database' +import { OrchestrationDb } from './db' +import { SCHEMA_VERSION } from './db/contract-constants' +import { isCurrentRunCoordinator } from './run-coordinator-authority' + +describe('Run coordinator authority migration', () => { + let directory: string | undefined + let db: OrchestrationDb | undefined + + afterEach(() => { + db?.close() + if (directory) { + rmSync(directory, { recursive: true, force: true }) + } + }) + + it('adds nullable authority identity to v30 Runs and backfills on same-pane use', () => { + directory = mkdtempSync(join(tmpdir(), 'orca-run-authority-')) + const dbPath = join(directory, 'orchestration.db') + const initial = new OrchestrationDb(dbPath) + const run = initial.createRun({ + objective: 'Retained v30 Run', + coordinatorHandle: 'term_old', + coordinatorPaneKey: 'tab_old:11111111-1111-4111-8111-111111111111' + }) + initial.close() + + const oldDb = new Database(dbPath) + oldDb.exec('DROP TRIGGER trg_runs_clear_stale_coordinator_authority') + oldDb.exec('ALTER TABLE runs DROP COLUMN coordinator_process_incarnation') + oldDb.exec('ALTER TABLE runs DROP COLUMN coordinator_host_scope') + oldDb.exec('ALTER TABLE runs DROP COLUMN coordinator_authority_revision') + oldDb.pragma('user_version = 30') + oldDb.close() + + db = new OrchestrationDb(dbPath) + expect(db.db.pragma('user_version', { simple: true })).toBe(SCHEMA_VERSION) + expect(db.getRun(run.id)).toMatchObject({ + coordinator_process_incarnation: null, + coordinator_host_scope: null, + coordinator_authority_revision: -1 + }) + const currentRun = db.createRun({ + objective: 'Created after v31 migration', + coordinatorHandle: 'term_current', + coordinatorPaneKey: 'tab_current:22222222-2222-4222-9222-222222222222', + coordinatorProcessIncarnation: 'current:incarnation-1', + coordinatorHostScope: JSON.stringify({ kind: 'local', hostId: 'local' }) + }) + expect(currentRun.coordinator_authority_revision).toBe(0) + expect( + isCurrentRunCoordinator(currentRun, { + handle: 'term_current', + paneKey: 'tab_current:22222222-2222-4222-9222-222222222222', + processIncarnation: 'current:incarnation-1', + hostScope: JSON.stringify({ kind: 'local', hostId: 'local' }) + }) + ).toBe(true) + expect( + isCurrentRunCoordinator(db.getRun(run.id)!, { + handle: 'term_replacement', + paneKey: 'tab_replacement:11111111-1111-4111-8111-111111111111', + processIncarnation: 'replacement:incarnation-1', + hostScope: JSON.stringify({ kind: 'local', hostId: 'local' }) + }) + ).toBe(false) + expect( + isCurrentRunCoordinator(db.getRun(run.id)!, { + handle: 'term_old', + paneKey: 'tab_replacement:11111111-1111-4111-8111-111111111111', + processIncarnation: 'replacement:incarnation-1', + hostScope: JSON.stringify({ kind: 'local', hostId: 'local' }) + }) + ).toBe(false) + + const rebound = db.bindRun({ + runId: run.id, + coordinatorHandle: 'term_reminted', + coordinatorPaneKey: 'tab_reminted:11111111-1111-4111-8111-111111111111', + coordinatorProcessIncarnation: 'pty:incarnation-1', + coordinatorHostScope: JSON.stringify({ kind: 'local', hostId: 'local' }), + authorityContinuity: true + }) + expect(rebound).toMatchObject({ + coordinator_handle: 'term_reminted', + coordinator_process_incarnation: 'pty:incarnation-1', + consumer_generation: run.consumer_generation + }) + }) + + it('clears stale authority when an older runtime changes only the binding columns', () => { + db = new OrchestrationDb(':memory:') + const run = db.createRun({ + objective: 'Mixed-version binding', + coordinatorHandle: 'term_old', + coordinatorPaneKey: 'tab_old:11111111-1111-4111-8111-111111111111', + coordinatorProcessIncarnation: 'pty-old:incarnation-1', + coordinatorHostScope: JSON.stringify({ kind: 'local', hostId: 'local' }) + }) + + db.db + .prepare( + `UPDATE runs + SET coordinator_handle = ?, coordinator_pane_key = ?, + consumer_generation = consumer_generation + 1 + WHERE id = ?` + ) + .run('term_v30', 'tab_v30:22222222-2222-4222-9222-222222222222', run.id) + + expect(db.getRun(run.id)).toMatchObject({ + coordinator_handle: 'term_v30', + coordinator_process_incarnation: null, + coordinator_host_scope: null, + coordinator_authority_revision: -1 + }) + }) +}) diff --git a/src/main/runtime/orchestration/run-coordinator-authority.ts b/src/main/runtime/orchestration/run-coordinator-authority.ts new file mode 100644 index 000000000000..8468987a05f8 --- /dev/null +++ b/src/main/runtime/orchestration/run-coordinator-authority.ts @@ -0,0 +1,32 @@ +import { isEquivalentPaneKey } from './db/pane-key-match' +import type { RunRow } from './types' + +export type RunCoordinatorIdentity = { + handle: string + paneKey: string | null + processIncarnation: string | null + hostScope: string | null +} + +export function isCurrentRunCoordinator(run: RunRow, identity: RunCoordinatorIdentity): boolean { + if (run.coordinator_authority_revision < 0) { + return false + } + const hasProcessAuthority = Boolean( + run.coordinator_process_incarnation || run.coordinator_host_scope + ) + if (hasProcessAuthority) { + return Boolean( + run.coordinator_process_incarnation && + identity.processIncarnation && + run.coordinator_process_incarnation === identity.processIncarnation && + run.coordinator_host_scope === identity.hostScope + ) + } + return Boolean( + run.coordinator_handle === identity.handle && + run.coordinator_pane_key && + identity.paneKey && + isEquivalentPaneKey(run.coordinator_pane_key, identity.paneKey) + ) +} diff --git a/src/main/runtime/orchestration/run-coordinator-handle-migration.test.ts b/src/main/runtime/orchestration/run-coordinator-handle-migration.test.ts index f15ad3c316a6..47ea51b01c2d 100644 --- a/src/main/runtime/orchestration/run-coordinator-handle-migration.test.ts +++ b/src/main/runtime/orchestration/run-coordinator-handle-migration.test.ts @@ -48,7 +48,8 @@ describe('Run coordinator handle history migration', () => { db.bindRun({ runId: run.id, coordinatorHandle: 'term_new', - coordinatorPaneKey: 'tab:leaf' + coordinatorPaneKey: 'tab:leaf', + authorityContinuity: true }) const late = db.insertMessage({ runId: run.id, diff --git a/src/main/runtime/orchestration/types.ts b/src/main/runtime/orchestration/types.ts index b34f69e3c229..a1bd79e0490f 100644 --- a/src/main/runtime/orchestration/types.ts +++ b/src/main/runtime/orchestration/types.ts @@ -46,6 +46,9 @@ export type RunRow = { home_database: string coordinator_handle: string | null coordinator_pane_key: string | null + coordinator_process_incarnation: string | null + coordinator_host_scope: string | null + coordinator_authority_revision: number consumer_generation: number legacy: number created_at: string diff --git a/src/main/runtime/orchestration/worker-terminal-process-liveness.ts b/src/main/runtime/orchestration/worker-terminal-process-liveness.ts index 67277644bfcd..eece3daedba7 100644 --- a/src/main/runtime/orchestration/worker-terminal-process-liveness.ts +++ b/src/main/runtime/orchestration/worker-terminal-process-liveness.ts @@ -40,11 +40,16 @@ export function classifyWorkerTerminalProcessIncarnation( processIncarnation: string, sessions: readonly PtyProcessInfo[] ): 'live' | 'exited' | 'unverifiable' { - const possibleMatches = sessions.filter((session) => + const directMatches = sessions.filter((session) => processIncarnation.startsWith(`${session.id}:`) ) + const legacyMatches = sessions.filter( + (session) => + !processIncarnation.startsWith(`${session.id}:`) && + processIncarnation.includes(`:${session.id}:`) + ) if ( - possibleMatches.some((session) => { + directMatches.some((session) => { const incarnationId = session.incarnationId if (!incarnationId || incarnationId !== incarnationId.trim()) { return false @@ -54,7 +59,10 @@ export function classifyWorkerTerminalProcessIncarnation( ) { return 'live' } - return possibleMatches.some( + if (legacyMatches.length > 0) { + return 'unverifiable' + } + return directMatches.some( (session) => !session.incarnationId || session.incarnationId !== session.incarnationId.trim() ) ? 'unverifiable' diff --git a/src/main/runtime/rpc/dispatcher-orchestration-caller-evidence.ts b/src/main/runtime/rpc/dispatcher-orchestration-caller-evidence.ts new file mode 100644 index 000000000000..03a898286122 --- /dev/null +++ b/src/main/runtime/rpc/dispatcher-orchestration-caller-evidence.ts @@ -0,0 +1,23 @@ +import type { OrchestrationCompatibilityEvidence } from '../../../shared/orchestration-compatibility-evidence' +import type { RpcRequest } from './core' + +const UNATTESTED_CALLER = Object.freeze({ paneKey: '', launchToken: '' }) + +export function resolveRpcCallerEvidence( + request: RpcRequest, + params: unknown, + legacyCoordinatorAuthority?: unknown +): OrchestrationCompatibilityEvidence | undefined { + if ( + request.orchestrationCompatibilityEvidence || + legacyCoordinatorAuthority || + !request.method.startsWith('orchestration.') + ) { + return request.orchestrationCompatibilityEvidence + } + const values = params as Record + const terminalHandle = [values.callerTerminalHandle, values.from, values.terminal].find( + (value): value is string => typeof value === 'string' + ) + return { terminalHandle: terminalHandle ?? '', ...UNATTESTED_CALLER } +} diff --git a/src/main/runtime/rpc/dispatcher.ts b/src/main/runtime/rpc/dispatcher.ts index 2ef1c235766a..9bbe8d6b6968 100644 --- a/src/main/runtime/rpc/dispatcher.ts +++ b/src/main/runtime/rpc/dispatcher.ts @@ -26,6 +26,7 @@ import { parseRpcRequestParams } from './dispatcher-request-parsing' import { routeDispatcherClientHostedBrowserRpc } from './dispatcher-client-browser-routing' import { needsLocalCallerFingerprint } from './dispatcher-caller-fingerprint' import { createDispatcherStreamingFeatureEmitter } from './dispatcher-streaming-feature-emitter' +import { resolveRpcCallerEvidence } from './dispatcher-orchestration-caller-evidence' export type DispatcherOptions = { runtime: OrcaRuntimeService; methods?: readonly RpcAnyMethod[] } @@ -108,6 +109,7 @@ export class RpcDispatcher { request, compatibility.legacyCoordinatorAuthority ) + const callerEvidence = resolveRpcCallerEvidence(request, effectiveParams, legacyCoordinator) const authenticatedCallerFingerprint = options?.authenticatedCallerFingerprint ?? (needsLocalCallerFingerprint(request, effectiveParams) @@ -129,7 +131,7 @@ export class RpcDispatcher { revalidateLegacyCoordinator: legacyCoordinator?.revalidate, orchestrationCompatibilityCallerAuthority: compatibility.orchestrationCompatibilityCallerAuthority, - orchestrationCompatibilityEvidence: request.orchestrationCompatibilityEvidence + orchestrationCompatibilityEvidence: callerEvidence }) } const result = await this.orchestrationMutations.run( @@ -217,6 +219,7 @@ export class RpcDispatcher { request, compatibility.legacyCoordinatorAuthority ) + const callerEvidence = resolveRpcCallerEvidence(request, effectiveParams, legacyCoordinator) const authenticatedCallerFingerprint = options?.authenticatedCallerFingerprint ?? (needsLocalCallerFingerprint(request, effectiveParams) @@ -247,7 +250,7 @@ export class RpcDispatcher { revalidateLegacyCoordinator: legacyCoordinator?.revalidate, orchestrationCompatibilityCallerAuthority: compatibility.orchestrationCompatibilityCallerAuthority, - orchestrationCompatibilityEvidence: request.orchestrationCompatibilityEvidence + orchestrationCompatibilityEvidence: callerEvidence }) } const result = await this.orchestrationMutations.run( diff --git a/src/main/runtime/rpc/methods/orchestration-ask.test.ts b/src/main/runtime/rpc/methods/orchestration-ask.test.ts index 18c1e9f415d1..cc326e2e468f 100644 --- a/src/main/runtime/rpc/methods/orchestration-ask.test.ts +++ b/src/main/runtime/rpc/methods/orchestration-ask.test.ts @@ -31,7 +31,7 @@ describe('orchestration RPC methods', () => { setup() const original = db.insertMessage({ from: 'a', - to: 'b', + to: `run:${activeRunId}`, subject: 'question', runId: activeRunId }) @@ -39,7 +39,7 @@ describe('orchestration RPC methods', () => { const result = (await call('orchestration.reply', { id: original.id, body: 'answer', - from: 'b' + from: 'term_coord' })) as { message: { to_handle: string; subject: string; thread_id: string; run_id: string } } diff --git a/src/main/runtime/rpc/methods/orchestration-caller-attestation.test.ts b/src/main/runtime/rpc/methods/orchestration-caller-attestation.test.ts new file mode 100644 index 000000000000..b86f11aed1d5 --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration-caller-attestation.test.ts @@ -0,0 +1,190 @@ +import { afterEach, describe, expect, it } from 'vitest' +import { ORCHESTRATION_CONTRACT_VERSION } from '../../../../shared/protocol-version' +import { RpcDispatcher } from '../dispatcher' +import { createOrchestrationRpcHarness } from './orchestration-rpc-test-harness' +import { ORCHESTRATION_METHODS } from './orchestration' +import { createRootDispatch } from '../../orchestration/db/root-dispatch-test-fixture' + +describe('orchestration coordinator caller attestation', () => { + const harness = createOrchestrationRpcHarness() + + afterEach(() => harness.cleanup()) + + it('rejects an identity-less caller that names the current coordinator handle', async () => { + const { db, runtime } = harness.setup() + const dispatcher = new RpcDispatcher({ runtime, methods: ORCHESTRATION_METHODS }) + const response = await dispatcher.dispatch({ + id: 'rpc_identity-less-coordinator', + authToken: 'test-token', + method: 'orchestration.taskCreate', + orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION, + params: { + spec: 'impersonated coordinator mutation', + run: db.getCurrentRunForPane(harness.coordinatorPaneKey)!.id, + callerTerminalHandle: 'term_coord' + } + }) + + expect(response).toMatchObject({ + ok: false, + error: { + code: 'consumer_fenced', + message: expect.stringContaining('authenticated identity from a live Orca agent terminal'), + data: { + effectsApplied: false, + nextSteps: expect.arrayContaining([ + expect.stringContaining('Omit --from'), + expect.stringContaining('copied terminal handle does not grant mutation authority') + ]) + } + } + }) + }) + + it.each([ + ['run-show', 'orchestration.runShow', (runId: string) => ({ id: runId, from: 'term_coord' })], + [ + 'task-list', + 'orchestration.taskList', + (runId: string) => ({ run: runId, callerTerminalHandle: 'term_coord' }) + ], + ['gate-list', 'orchestration.gateList', (runId: string) => ({ run: runId, from: 'term_coord' })] + ])('marks an identity-less explicit %s read as non-owner', async (_label, method, params) => { + const { db, runtime } = harness.setup() + const runId = db.getCurrentRunForPane(harness.coordinatorPaneKey)!.id + const dispatcher = new RpcDispatcher({ runtime, methods: ORCHESTRATION_METHODS }) + const response = await dispatcher.dispatch({ + id: `rpc_identity-less-${method}`, + authToken: 'test-token', + method, + orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION, + params: params(runId) + }) + + expect(response).toMatchObject({ + ok: true, + result: { binding: { currentConsumer: false } } + }) + }) + + it('rejects an identity-less sender that copies the current coordinator handle', async () => { + const { db, runtime } = harness.setup() + const runId = db.getCurrentRunForPane(harness.coordinatorPaneKey)!.id + const dispatcher = new RpcDispatcher({ runtime, methods: ORCHESTRATION_METHODS }) + const response = await dispatcher.dispatch({ + id: 'rpc_identity-less-send', + authToken: 'test-token', + method: 'orchestration.send', + orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION, + params: { + from: 'term_coord', + to: `run:${runId}`, + subject: 'impersonated coordinator control mail' + } + }) + + expect(response).toMatchObject({ + ok: false, + error: { code: 'consumer_fenced', data: { effectsApplied: false } } + }) + expect(db.getInbox(100)).toHaveLength(0) + }) + + it('does not let a bogus capability bypass attestation for a capability-less Dispatch', async () => { + const { db, runtime } = harness.setup() + const task = db.createTask({ spec: 'manual Dispatch work' }) + const dispatch = createRootDispatch( + db, + task.id, + 'term_worker', + 'tab_worker:bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb' + ) + const dispatcher = new RpcDispatcher({ runtime, methods: ORCHESTRATION_METHODS }) + const response = await dispatcher.dispatch({ + id: 'rpc_bogus-capability-send', + authToken: 'test-token', + method: 'orchestration.send', + orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION, + orchestrationCapability: 'dcap_bogus', + params: { + from: 'term_worker', + subject: 'forged completion', + type: 'worker_done', + payload: JSON.stringify({ + taskId: task.id, + dispatchId: dispatch.id, + outcome: 'succeeded' + }) + } + }) + + expect(response).toMatchObject({ + ok: false, + error: { code: 'consumer_fenced', data: { effectsApplied: false } } + }) + expect(db.getTask(task.id)?.status).toBe('dispatched') + expect(db.getInbox(100)).toHaveLength(0) + }) + + it('rejects an identity-less non-question reply that copies the current coordinator handle', async () => { + const { db, runtime } = harness.setup() + const runId = db.getCurrentRunForPane(harness.coordinatorPaneKey)!.id + const original = db.insertMessage({ + runId, + from: 'term_worker', + to: 'term_coord', + subject: 'worker status' + }) + const dispatcher = new RpcDispatcher({ runtime, methods: ORCHESTRATION_METHODS }) + const response = await dispatcher.dispatch({ + id: 'rpc_identity-less-reply', + authToken: 'test-token', + method: 'orchestration.reply', + orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION, + params: { id: original.id, body: 'impersonated reply', from: 'term_coord' } + }) + + expect(response).toMatchObject({ + ok: false, + error: { code: 'consumer_fenced', data: { effectsApplied: false } } + }) + expect(db.getMessageById(original.id)?.read).toBe(0) + expect(db.getInbox(100)).toHaveLength(1) + }) + + it('rejects an attested coordinator replying to another Run mailbox', async () => { + const { db, runtime } = harness.setup() + const foreignRun = db.createRun({ + objective: 'Foreign Run', + coordinatorHandle: 'term_foreign', + coordinatorPaneKey: 'tab_foreign:cccccccc-cccc-4ccc-8ccc-cccccccccccc', + coordinatorProcessIncarnation: 'runtime_test:term_foreign:1', + coordinatorHostScope: JSON.stringify({ kind: 'local', hostId: 'local' }) + }) + const original = db.insertMessage({ + runId: foreignRun.id, + from: 'term_foreign_worker', + to: `run:${foreignRun.id}`, + subject: 'foreign status' + }) + const dispatcher = new RpcDispatcher({ runtime, methods: ORCHESTRATION_METHODS }) + const response = await dispatcher.dispatch({ + id: 'rpc_cross-run-non-question-reply', + authToken: 'test-token', + method: 'orchestration.reply', + orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION, + orchestrationCompatibilityEvidence: { + terminalHandle: 'term_coord', + paneKey: harness.coordinatorPaneKey, + launchToken: 'test-launch-token' + }, + params: { id: original.id, body: 'forged reply', from: 'term_coord' } + }) + + expect(response).toMatchObject({ + ok: false, + error: { code: 'consumer_fenced' } + }) + expect(db.getMessageById(original.id)?.read).toBe(0) + }) +}) diff --git a/src/main/runtime/rpc/methods/orchestration-check.test.ts b/src/main/runtime/rpc/methods/orchestration-check.test.ts index 331e74487344..46ec8de7dce6 100644 --- a/src/main/runtime/rpc/methods/orchestration-check.test.ts +++ b/src/main/runtime/rpc/methods/orchestration-check.test.ts @@ -120,6 +120,15 @@ describe('orchestration RPC methods', () => { it('uses the stable pane identity when the coordinator handle was reminted', async () => { setup() + db.bindRun({ + runId: activeRunId!, + coordinatorHandle: 'term_coord', + coordinatorPaneKey, + coordinatorProcessIncarnation: 'pty-coordinator:incarnation-1' + }) + vi.mocked(runtime.getTerminalProcessIncarnation).mockReturnValue( + 'pty-coordinator:incarnation-1' + ) db.insertMessage({ from: 'term_worker', to: `run:${activeRunId}`, diff --git a/src/main/runtime/rpc/methods/orchestration-composed-workers.test.ts b/src/main/runtime/rpc/methods/orchestration-composed-workers.test.ts index 10dbfa97a53a..7925f8488fe9 100644 --- a/src/main/runtime/rpc/methods/orchestration-composed-workers.test.ts +++ b/src/main/runtime/rpc/methods/orchestration-composed-workers.test.ts @@ -56,7 +56,11 @@ describe('orchestration RPC methods', () => { exitCode: null }) vi.mocked(runtime.getTerminalProcessIncarnation).mockImplementation((handle) => - handle === 'term_worker' ? 'runtime_test:term_worker:1' : null + handle === 'term_worker' + ? 'runtime_test:term_worker:1' + : handle === 'term_coord' + ? 'runtime_test:term_coord:1' + : null ) vi.spyOn(runtime, 'getTerminalOrchestrationCliCommand').mockReturnValue('orca') vi.spyOn(runtime, 'sendTerminalAgentPrompt').mockResolvedValue({ @@ -101,7 +105,7 @@ describe('orchestration RPC methods', () => { expect(db.getDispatchContext(task.id)).toBeUndefined() }) - it('deliberately permits present but unverifiable restored-terminal evidence', async () => { + it('rejects restored-terminal evidence that names a different caller', async () => { setup() mockCurrentWorkerStart() // Restored/adopted terminals have no launch token, so verification returns null; this @@ -115,14 +119,14 @@ describe('orchestration RPC methods', () => { } } - const result = (await call('orchestration.workerStart', { - task: task.id, - from: 'term_coord', - agent: 'codex' - })) as { state: string } - - expect(result.state).toBe('ready') - expect(db.getDispatchContext(task.id)).toBeDefined() + await expect( + call('orchestration.workerStart', { + task: task.id, + from: 'term_coord', + agent: 'codex' + }) + ).rejects.toMatchObject({ code: 'consumer_fenced' }) + expect(db.getDispatchContext(task.id)).toBeUndefined() }) it('starts a fresh agent in the coordinator current worktree', async () => { diff --git a/src/main/runtime/rpc/methods/orchestration-coordinator-caller.ts b/src/main/runtime/rpc/methods/orchestration-coordinator-caller.ts new file mode 100644 index 000000000000..617ebfb31e3a --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration-coordinator-caller.ts @@ -0,0 +1,51 @@ +import { isCurrentRunCoordinator } from '../../orchestration/run-coordinator-authority' +import type { RunRow } from '../../orchestration/types' +import type { OrcaRuntimeService } from '../../orca-runtime' +import type { OrchestrationCompatibilityEvidence } from '../../../../shared/orchestration-compatibility-evidence' + +export function resolveRunCoordinatorIdentity( + runtime: OrcaRuntimeService, + handle: string, + paneKey = runtime.getTerminalPaneKey(handle) +) { + const authority = runtime.getOrchestrationDispatchAuthority(handle) + return { + handle, + paneKey, + processIncarnation: + authority?.processIncarnation ?? runtime.getTerminalProcessIncarnation(handle), + hostScope: authority?.hostScope ? JSON.stringify(authority.hostScope) : null + } +} + +export function isCallerCurrentRunCoordinator( + runtime: OrcaRuntimeService, + run: RunRow, + handle: string, + paneKey?: string | null +): boolean { + return isCurrentRunCoordinator( + run, + resolveRunCoordinatorIdentity(runtime, handle, paneKey ?? runtime.getTerminalPaneKey(handle)) + ) +} + +export function resolveAttestedRunCoordinatorPane( + runtime: OrcaRuntimeService, + run: RunRow, + handle: string, + evidence?: OrchestrationCompatibilityEvidence +): string | null { + const caller = runtime.verifyOrchestrationCompatibilityCaller(evidence) + if (!caller || caller.terminalHandle !== handle) { + return null + } + return isCurrentRunCoordinator(run, { + handle, + paneKey: caller.paneKey, + processIncarnation: caller.processIncarnation, + hostScope: JSON.stringify(caller.hostScope) + }) + ? caller.paneKey + : null +} diff --git a/src/main/runtime/rpc/methods/orchestration-federation-control-mail.test.ts b/src/main/runtime/rpc/methods/orchestration-federation-control-mail.test.ts index 401318d82c05..07e995e693b0 100644 --- a/src/main/runtime/rpc/methods/orchestration-federation-control-mail.test.ts +++ b/src/main/runtime/rpc/methods/orchestration-federation-control-mail.test.ts @@ -78,6 +78,19 @@ describe('orchestration federation control mail', () => { vi.spyOn(homeRuntime, 'getTerminalPaneKey').mockImplementation((handle) => handle === 'term_coord' ? coordinatorPaneKey : null ) + vi.spyOn(homeRuntime, 'verifyOrchestrationCompatibilityCaller').mockImplementation((evidence) => + evidence?.terminalHandle === 'term_coord' && + evidence.paneKey === coordinatorPaneKey && + evidence.launchToken === 'coordinator-launch-token' + ? { + hostScope: { kind: 'local', hostId: 'local' }, + paneKey: coordinatorPaneKey, + terminalHandle: 'term_coord', + processIncarnation: 'coordinator-process:1', + launchTokenHash: 'coordinator-launch-hash' + } + : null + ) homeDispatcher = new RpcDispatcher({ runtime: homeRuntime, methods: ORCHESTRATION_METHODS @@ -143,6 +156,11 @@ describe('orchestration federation control mail', () => { authToken: 'coordinator-token', orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION, orchestrationRequestId: 'send-control-request', + orchestrationCompatibilityEvidence: { + terminalHandle: 'term_coord', + paneKey: coordinatorPaneKey, + launchToken: 'coordinator-launch-token' + }, method: 'orchestration.send', params: { from: 'term_coord', @@ -217,6 +235,11 @@ describe('orchestration federation control mail', () => { authToken: 'coordinator-token', orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION, orchestrationRequestId: 'send-stale-control-request', + orchestrationCompatibilityEvidence: { + terminalHandle: 'term_coord', + paneKey: coordinatorPaneKey, + launchToken: 'coordinator-launch-token' + }, method: 'orchestration.send', params: { from: 'term_coord', diff --git a/src/main/runtime/rpc/methods/orchestration-federation-lifecycle-settlement.test.ts b/src/main/runtime/rpc/methods/orchestration-federation-lifecycle-settlement.test.ts index a54eca1fd845..47baeb783324 100644 --- a/src/main/runtime/rpc/methods/orchestration-federation-lifecycle-settlement.test.ts +++ b/src/main/runtime/rpc/methods/orchestration-federation-lifecycle-settlement.test.ts @@ -11,7 +11,10 @@ import type { OrchestrationEnvironmentTransport } from '../../orchestration/envi import { waitForFederatedLifecycleSettlement } from '../../orchestration/federation-lifecycle-settlement' import { RpcDispatcher } from '../dispatcher' import { ORCHESTRATION_METHODS } from './orchestration' -import { createFederationWorkerStartRequest as startRequest } from './orchestration-federation-test-request' +import { + createFederationWorkerStartRequest as startRequest, + mockFederationCoordinatorAttestation +} from './orchestration-federation-test-request' describe('orchestration federation lifecycle settlement', () => { let homeDb: OrchestrationDb @@ -72,9 +75,7 @@ describe('orchestration federation lifecycle settlement', () => { }) homeRuntime.setOrchestrationDb(homeDb) homeDispatcher = new RpcDispatcher({ runtime: homeRuntime, methods: ORCHESTRATION_METHODS }) - vi.spyOn(homeRuntime, 'getTerminalPaneKey').mockReturnValue( - 'tab_coord:aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa' - ) + mockFederationCoordinatorAttestation(homeRuntime) configureWorkerRuntime() }) diff --git a/src/main/runtime/rpc/methods/orchestration-federation-output.test.ts b/src/main/runtime/rpc/methods/orchestration-federation-output.test.ts index adb2cc5459a6..5fcf3f95ad54 100644 --- a/src/main/runtime/rpc/methods/orchestration-federation-output.test.ts +++ b/src/main/runtime/rpc/methods/orchestration-federation-output.test.ts @@ -10,6 +10,10 @@ import type { OrchestrationEnvironmentTransport } from '../../orchestration/envi import type { RpcRequest } from '../core' import { RpcDispatcher } from '../dispatcher' import { ORCHESTRATION_METHODS } from './orchestration' +import { + FEDERATION_COORDINATOR_EVIDENCE, + mockFederationCoordinatorAttestation +} from './orchestration-federation-test-request' describe('orchestration federated worker output', () => { const databases: OrchestrationDb[] = [] @@ -73,9 +77,7 @@ describe('orchestration federated worker output', () => { runtime: homeRuntime, methods: ORCHESTRATION_METHODS }) - vi.spyOn(homeRuntime, 'getTerminalPaneKey').mockImplementation((handle) => - handle === 'term_coord' ? 'tab_coord:aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa' : null - ) + mockFederationCoordinatorAttestation(homeRuntime) configureWorkerRuntime(workerRuntime) }) @@ -101,6 +103,7 @@ describe('orchestration federated worker output', () => { authToken: 'coordinator-token', orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION, orchestrationRequestId: 'request_windows_worker', + orchestrationCompatibilityEvidence: FEDERATION_COORDINATOR_EVIDENCE, method: 'orchestration.workerStart', params: { task: taskId, diff --git a/src/main/runtime/rpc/methods/orchestration-federation-test-request.ts b/src/main/runtime/rpc/methods/orchestration-federation-test-request.ts index 7d0047753107..8c1b8fd2ae9c 100644 --- a/src/main/runtime/rpc/methods/orchestration-federation-test-request.ts +++ b/src/main/runtime/rpc/methods/orchestration-federation-test-request.ts @@ -1,6 +1,36 @@ +import { vi } from 'vitest' import { ORCHESTRATION_CONTRACT_VERSION } from '../../../../shared/protocol-version' +import type { OrchestrationCompatibilityEvidence } from '../../../../shared/orchestration-compatibility-evidence' +import type { OrcaRuntimeService } from '../../orca-runtime' import type { RpcRequest } from '../core' +export const FEDERATION_COORDINATOR_EVIDENCE = { + terminalHandle: 'term_coord', + paneKey: 'tab_coord:aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa', + launchToken: 'coordinator-launch-token' +} as const satisfies OrchestrationCompatibilityEvidence + +export function mockFederationCoordinatorAttestation(runtime: OrcaRuntimeService): void { + vi.spyOn(runtime, 'getTerminalPaneKey').mockImplementation((handle) => + handle === FEDERATION_COORDINATOR_EVIDENCE.terminalHandle + ? FEDERATION_COORDINATOR_EVIDENCE.paneKey + : null + ) + vi.spyOn(runtime, 'verifyOrchestrationCompatibilityCaller').mockImplementation((evidence) => + evidence?.terminalHandle === FEDERATION_COORDINATOR_EVIDENCE.terminalHandle && + evidence.paneKey === FEDERATION_COORDINATOR_EVIDENCE.paneKey + ? { + terminalHandle: evidence.terminalHandle, + paneKey: evidence.paneKey, + processIncarnation: 'home_runtime:coordinator:1', + hostScope: { kind: 'local', hostId: 'local' }, + launchTokenHash: 'test-token-hash', + terminalProvenance: 'current_runtime' + } + : null + ) +} + export function createFederationWorkerStartRequest( taskId: string, overrides: Record = {} @@ -10,6 +40,7 @@ export function createFederationWorkerStartRequest( authToken: 'coordinator-token', orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION, orchestrationRequestId: 'request_windows_worker', + orchestrationCompatibilityEvidence: FEDERATION_COORDINATOR_EVIDENCE, method: 'orchestration.workerStart', params: { task: taskId, diff --git a/src/main/runtime/rpc/methods/orchestration-federation.test.ts b/src/main/runtime/rpc/methods/orchestration-federation.test.ts index 59dd8a7a5f8f..98871cb0e068 100644 --- a/src/main/runtime/rpc/methods/orchestration-federation.test.ts +++ b/src/main/runtime/rpc/methods/orchestration-federation.test.ts @@ -9,7 +9,11 @@ import { OrchestrationDb } from '../../orchestration/db' import type { OrchestrationEnvironmentTransport } from '../../orchestration/environment-transport' import { RpcDispatcher } from '../dispatcher' import { ORCHESTRATION_METHODS } from './orchestration' -import { createFederationWorkerStartRequest as startRequest } from './orchestration-federation-test-request' +import { + createFederationWorkerStartRequest as startRequest, + FEDERATION_COORDINATOR_EVIDENCE, + mockFederationCoordinatorAttestation +} from './orchestration-federation-test-request' describe('orchestration federation', () => { const databases: OrchestrationDb[] = [] @@ -75,9 +79,7 @@ describe('orchestration federation', () => { runtime: homeRuntime, methods: ORCHESTRATION_METHODS }) - vi.spyOn(homeRuntime, 'getTerminalPaneKey').mockImplementation((handle) => - handle === 'term_coord' ? 'tab_coord:aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa' : null - ) + mockFederationCoordinatorAttestation(homeRuntime) configureWorkerRuntime(workerRuntime) }) @@ -415,6 +417,7 @@ describe('orchestration federation', () => { authToken: 'coordinator-token', orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION, orchestrationRequestId: 'home_reply_request', + orchestrationCompatibilityEvidence: FEDERATION_COORDINATOR_EVIDENCE, method: 'orchestration.reply', params: { id: question!.id, @@ -472,6 +475,7 @@ describe('orchestration federation', () => { authToken: 'coordinator-token', orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION, orchestrationRequestId: 'home_late_reply_request', + orchestrationCompatibilityEvidence: FEDERATION_COORDINATOR_EVIDENCE, method: 'orchestration.reply', params: { id: questionId, body: 'yes', from: 'term_coord' } }) diff --git a/src/main/runtime/rpc/methods/orchestration-gate-run-authorization.test.ts b/src/main/runtime/rpc/methods/orchestration-gate-run-authorization.test.ts index ee18c3661857..89e824ec5869 100644 --- a/src/main/runtime/rpc/methods/orchestration-gate-run-authorization.test.ts +++ b/src/main/runtime/rpc/methods/orchestration-gate-run-authorization.test.ts @@ -63,7 +63,7 @@ describe('orchestration gate Run authorization', () => { it('G2 refuses an unattested caller creating a gate on another Run task', async () => { const harness = createHarness() - const runA = bindCurrentCoordinatorRun(harness) + bindCurrentCoordinatorRun(harness) const before = adoptedTaskState(harness) const response = await harness.dispatcher.dispatch( @@ -78,8 +78,8 @@ describe('orchestration gate Run authorization', () => { expect(response).toMatchObject({ ok: false, error: { - code: 'task_not_found', - message: `Task ${harness.taskId} was not found in Run ${runA}.` + code: 'consumer_fenced', + data: { effectsApplied: false } } }) expect(adoptedTaskState(harness)).toEqual(before) @@ -214,7 +214,10 @@ describe('orchestration gate Run authorization', () => { ) ) - expect(response).toMatchObject({ ok: false, error: { message: `Gate not found: ${gate.id}` } }) + expect(response).toMatchObject({ + ok: false, + error: { code: 'consumer_fenced', data: { effectsApplied: false } } + }) expect(harness.db.getGate(gate.id)?.status).toBe('pending') expect(harness.db.getTask(harness.taskId)?.status).toBe('blocked') }) @@ -237,7 +240,12 @@ describe('orchestration gate Run authorization', () => { expect(response).toMatchObject({ ok: true, - result: { runId: runA, count: 1, gates: [{ id: ownGate.id }] } + result: { + runId: runA, + binding: { currentConsumer: true }, + count: 1, + gates: [{ id: ownGate.id }] + } }) }) diff --git a/src/main/runtime/rpc/methods/orchestration-gates.ts b/src/main/runtime/rpc/methods/orchestration-gates.ts index 1d9c7622fe9c..e6b27b64fa7e 100644 --- a/src/main/runtime/rpc/methods/orchestration-gates.ts +++ b/src/main/runtime/rpc/methods/orchestration-gates.ts @@ -5,6 +5,7 @@ import type { GateStatus } from '../../orchestration/db' import { Coordinator } from '../../orchestration/coordinator' import { resolveRunScope } from './orchestration-run-scope' import { OrchestrationError } from '../../orchestration/orchestration-error' +import { resolveAttestedRunCoordinatorPane } from './orchestration-coordinator-caller' // Why: the coordinator instance is stored at module scope so orchestration.runStop // can signal it to halt. Only one coordinator can run at a time (enforced by @@ -202,7 +203,27 @@ export const ORCHESTRATION_GATE_METHODS: RpcMethod[] = [ status: params.status as GateStatus }) .filter((gate) => gate.run_id === run.id) - return { runId: run.id, gates, count: gates.length } + const callerPaneKey = params.from + ? resolveAttestedRunCoordinatorPane( + runtime, + run, + params.from, + orchestrationCompatibilityEvidence + ) + : null + return { + runId: run.id, + binding: { + currentConsumer: Boolean( + params.from && + run.legacy === 0 && + callerPaneKey !== null && + db.getCurrentRunForPane(callerPaneKey)?.id === run.id + ) + }, + gates, + count: gates.length + } } }) ] diff --git a/src/main/runtime/rpc/methods/orchestration-recipient-routing.test.ts b/src/main/runtime/rpc/methods/orchestration-recipient-routing.test.ts index 41d10266dee4..24d3f745ca93 100644 --- a/src/main/runtime/rpc/methods/orchestration-recipient-routing.test.ts +++ b/src/main/runtime/rpc/methods/orchestration-recipient-routing.test.ts @@ -6,7 +6,10 @@ import type { OrcaRuntimeService } from '../../orca-runtime' import type { RpcContext, RpcRequest } from '../core' import { RpcDispatcher } from '../dispatcher' import { ORCHESTRATION_METHODS } from './orchestration' -import { createOrchestrationRpcHarness } from './orchestration-rpc-test-harness' +import { + COORDINATOR_PANE_KEY, + createOrchestrationRpcHarness +} from './orchestration-rpc-test-harness' import { createRootDispatch } from '../../orchestration/db/root-dispatch-test-fixture' type SendWarning = { code: string; recipient: string; message: string } @@ -249,7 +252,8 @@ describe('orchestration recipient routing oracle', () => { db.bindRun({ runId: senderRunId, coordinatorHandle: 'term_current', - coordinatorPaneKey: 'tab_current:leaf_current' + coordinatorPaneKey: 'tab_current:leaf_current', + authorityContinuity: true }) mockTerminalPaneKeys((handle) => handle === 'term_current' ? 'tab_current:leaf_current' : null @@ -330,7 +334,8 @@ describe('orchestration recipient routing oracle', () => { db.bindRun({ runId: foreignRun.id, coordinatorHandle: 'term_foreign_second', - coordinatorPaneKey: 'tab_foreign_second:leaf_foreign_second' + coordinatorPaneKey: 'tab_foreign_second:leaf_foreign_second', + authorityContinuity: true }) vi.spyOn(runtime, 'listTerminals').mockResolvedValue({ terminals: [ @@ -363,12 +368,14 @@ describe('orchestration recipient routing oracle', () => { db.bindRun({ runId: senderRunId, coordinatorHandle: 'term_middle', - coordinatorPaneKey: 'tab_middle:leaf_middle' + coordinatorPaneKey: 'tab_middle:leaf_middle', + authorityContinuity: true }) db.bindRun({ runId: senderRunId, coordinatorHandle: 'term_sender', - coordinatorPaneKey: 'tab_sender:leaf_sender' + coordinatorPaneKey: 'tab_sender:leaf_sender', + authorityContinuity: true }) vi.spyOn(runtime, 'listTerminals').mockResolvedValue({ terminals: [ @@ -578,6 +585,11 @@ function request( method: 'orchestration.send', params: { from: 'term_coord', to, subject: 'retry', ...extraParams }, orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION, - orchestrationRequestId: requestId + orchestrationRequestId: requestId, + orchestrationCompatibilityEvidence: { + terminalHandle: 'term_coord', + paneKey: COORDINATOR_PANE_KEY, + launchToken: 'coordinator-launch-token' + } } } diff --git a/src/main/runtime/rpc/methods/orchestration-rpc-test-harness.ts b/src/main/runtime/rpc/methods/orchestration-rpc-test-harness.ts index b0a77bfa29fe..070e88e2a608 100644 --- a/src/main/runtime/rpc/methods/orchestration-rpc-test-harness.ts +++ b/src/main/runtime/rpc/methods/orchestration-rpc-test-harness.ts @@ -19,6 +19,7 @@ export function createOrchestrationRpcHarness() { let dbOpen = false let runtime: OrcaRuntimeService let activeRunId: string | undefined + let defaultCtx: RpcContext const coordinatorPaneKey = COORDINATOR_PANE_KEY @@ -36,6 +37,28 @@ export function createOrchestrationRpcHarness() { vi.spyOn(runtime, 'getTerminalProcessIncarnation').mockImplementation((handle) => handle.startsWith('term_') ? `runtime_test:${handle}:1` : null ) + vi.spyOn(runtime, 'verifyOrchestrationCompatibilityCaller').mockImplementation((evidence) => { + const terminalHandle = evidence?.terminalHandle + const paneKey = terminalHandle ? runtime.getTerminalPaneKey(terminalHandle) : null + const terminalAuthority = terminalHandle + ? runtime.getOrchestrationDispatchAuthority(terminalHandle) + : null + const processIncarnation = terminalHandle + ? (terminalAuthority?.processIncarnation ?? + runtime.getTerminalProcessIncarnation(terminalHandle)) + : null + if (!terminalHandle || !paneKey || !processIncarnation || evidence.paneKey !== paneKey) { + return null + } + return { + terminalHandle, + paneKey, + processIncarnation, + launchTokenHash: 'test-launch-token-hash', + hostScope: terminalAuthority?.hostScope ?? { kind: 'local', hostId: 'local' }, + terminalProvenance: 'current_runtime' + } + }) if (withBoundRun) { activeRunId = db.createRun({ objective: 'Test Run', @@ -51,7 +74,8 @@ export function createOrchestrationRpcHarness() { } else { activeRunId = undefined } - return { db, runtime, ctx: { runtime }, activeRunId } + defaultCtx = { runtime } + return { db, runtime, ctx: defaultCtx, activeRunId } } function cleanup(): void { @@ -96,7 +120,27 @@ export function createOrchestrationRpcHarness() { } } const parsed = method.params ? method.params.parse(scopedParams) : undefined - return method.handler(parsed, ctx) + const callerHandle = + typeof scopedParams.callerTerminalHandle === 'string' + ? scopedParams.callerTerminalHandle + : typeof scopedParams.from === 'string' + ? scopedParams.from + : typeof scopedParams.terminal === 'string' + ? scopedParams.terminal + : undefined + const callerPaneKey = callerHandle ? runtime.getTerminalPaneKey(callerHandle) : null + const effectiveCtx = + ctx === defaultCtx && callerHandle && callerPaneKey + ? { + ...ctx, + orchestrationCompatibilityEvidence: { + terminalHandle: callerHandle, + paneKey: callerPaneKey, + launchToken: 'test-launch-token' + } + } + : ctx + return method.handler(parsed, effectiveCtx) } return { coordinatorPaneKey, setup, cleanup, findMethod, call } diff --git a/src/main/runtime/rpc/methods/orchestration-run-coordinator-observation.ts b/src/main/runtime/rpc/methods/orchestration-run-coordinator-observation.ts new file mode 100644 index 000000000000..40b5f1ca6d3d --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration-run-coordinator-observation.ts @@ -0,0 +1,33 @@ +import type { RunCoordinatorIdentity } from '../../orchestration/run-coordinator-authority' +import type { RunRow } from '../../orchestration/types' +import type { OrcaRuntimeService } from '../../orca-runtime' + +export async function observeRunCoordinator( + runtime: OrcaRuntimeService, + run: RunRow, + resolvedIdentity?: RunCoordinatorIdentity | null +) { + let status: 'live' | 'unverifiable' | 'exited' = 'unverifiable' + const processIncarnation = + run.coordinator_process_incarnation ?? resolvedIdentity?.processIncarnation + const hostScope = run.coordinator_host_scope ?? resolvedIdentity?.hostScope + if (processIncarnation && hostScope) { + status = await runtime.inspectTerminalProcessIncarnationLiveness(processIncarnation, hostScope) + } else if (run.coordinator_handle) { + const verdict = runtime.getTerminalLivenessVerdict(run.coordinator_handle) + if (runtime.getLiveTerminalPaneKey(run.coordinator_handle) || verdict?.status === 'live') { + status = 'live' + } else if (verdict?.status === 'unverifiable') { + status = 'unverifiable' + } else if (verdict?.status === 'exited') { + status = 'exited' + } + } + return { + coordinatorHandle: run.coordinator_handle, + coordinatorPaneKey: run.coordinator_pane_key, + coordinatorProcessIncarnation: run.coordinator_process_incarnation, + coordinatorHostScope: run.coordinator_host_scope, + status + } +} diff --git a/src/main/runtime/rpc/methods/orchestration-run-remint-authority.test.ts b/src/main/runtime/rpc/methods/orchestration-run-remint-authority.test.ts new file mode 100644 index 000000000000..cfebb7d578cc --- /dev/null +++ b/src/main/runtime/rpc/methods/orchestration-run-remint-authority.test.ts @@ -0,0 +1,67 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import type { RpcContext } from '../core' +import { createOrchestrationRpcHarness } from './orchestration-rpc-test-harness' + +describe('orchestration Run remint authority', () => { + const harness = createOrchestrationRpcHarness() + + afterEach(() => harness.cleanup()) + + it('preserves a same-process remint when the agent hook still reports the old pane', async () => { + const { db, runtime } = harness.setup(false) + const oldPane = 'tab_old:11111111-1111-4111-8111-111111111111' + const remintedPane = 'tab_new:11111111-1111-4111-8111-111111111111' + vi.spyOn(runtime, 'getTerminalPaneKey').mockImplementation((handle) => + handle === 'term_old' ? oldPane : remintedPane + ) + vi.spyOn(runtime, 'getOrchestrationDispatchAuthority').mockImplementation((handle) => ({ + runtimeId: 'runtime_test', + terminalHandle: handle, + ptyId: 'pty_coord', + worktreeId: 'folder:workspace', + processIncarnation: 'pty_coord:incarnation-1', + paneKey: handle === 'term_old' ? oldPane : remintedPane, + launchTokenHash: 'current-launch-hash', + hostScope: { kind: 'local', hostId: 'local' } + })) + const created = db.createRun({ + objective: 'Survive stale hook evidence', + coordinatorHandle: 'term_old', + coordinatorPaneKey: oldPane, + coordinatorProcessIncarnation: 'pty_coord:incarnation-1', + coordinatorHostScope: JSON.stringify({ kind: 'local', hostId: 'local' }) + }) + const ctx: RpcContext = { + runtime, + orchestrationCompatibilityEvidence: { + terminalHandle: 'term_reminted', + paneKey: remintedPane, + launchToken: 'current-launch' + } + } + vi.spyOn(runtime, 'verifyOrchestrationCompatibilityCaller').mockImplementation( + (_evidence, options) => + options?.currentRuntimeLaunchSufficient + ? { + terminalHandle: 'term_reminted', + paneKey: remintedPane, + processIncarnation: 'pty_coord:incarnation-1', + launchTokenHash: 'current-launch-hash', + hostScope: { kind: 'local', hostId: 'local' }, + terminalProvenance: 'current_runtime' + } + : null + ) + + const rebound = (await harness.call( + 'orchestration.runUse', + { id: created.id, from: 'term_reminted' }, + ctx + )) as { run: { coordinator_handle: string; consumer_generation: number } } + + expect(rebound.run).toMatchObject({ + coordinator_handle: 'term_reminted', + consumer_generation: created.consumer_generation + }) + }) +}) diff --git a/src/main/runtime/rpc/methods/orchestration-run-scope.ts b/src/main/runtime/rpc/methods/orchestration-run-scope.ts index cdf6968bcbc5..fb89029a4e1d 100644 --- a/src/main/runtime/rpc/methods/orchestration-run-scope.ts +++ b/src/main/runtime/rpc/methods/orchestration-run-scope.ts @@ -6,6 +6,7 @@ import type { OrcaRuntimeService, OrchestrationCompatibilityCallerAuthority } from '../../orca-runtime' +import { isCallerCurrentRunCoordinator } from './orchestration-coordinator-caller' export type RunScopeParams = { runId?: string @@ -21,13 +22,40 @@ export type RunScopeParams = { export function assertCallerHandleMatchesEvidence( runtime: OrcaRuntimeService, callerTerminalHandle: string, - callerEvidence?: OrchestrationCompatibilityEvidence + callerEvidence?: OrchestrationCompatibilityEvidence, + options: { + callerAuthority?: OrchestrationCompatibilityCallerAuthority + allowLegacyAuthority?: boolean + } = {} ): void { - if (!callerEvidence) { + if (callerEvidence && callerEvidence.terminalHandle !== callerTerminalHandle) { + throw new OrchestrationError( + 'consumer_fenced', + `This terminal is attested as ${callerEvidence.terminalHandle} and cannot act as ${callerTerminalHandle}.`, + { effectsApplied: false } + ) + } + // Why: trusted in-process handlers omit evidence; public RPCs receive an invalid sentinel. + if (!callerEvidence || options.allowLegacyAuthority) { return } - const attested = runtime.verifyOrchestrationCompatibilityCaller(callerEvidence) - if (attested && attested.terminalHandle !== callerTerminalHandle) { + const attested = + options.callerAuthority ?? runtime.verifyOrchestrationCompatibilityCaller(callerEvidence) + if (!attested) { + throw new OrchestrationError( + 'consumer_fenced', + 'Orchestration mutations require authenticated identity from a live Orca agent terminal. No effects were applied.', + { + effectsApplied: false, + nextSteps: [ + 'Run the command inside the invoking agent terminal with the version-matched Orca CLI.', + 'Omit --from; Orca resolves and attests the invoking agent terminal automatically.', + 'Use explicit Run reads for inspection; a copied terminal handle does not grant mutation authority.' + ] + } + ) + } + if (attested.terminalHandle !== callerTerminalHandle) { throw new OrchestrationError( 'consumer_fenced', `This terminal is attested as ${attested.terminalHandle} and cannot act as ${callerTerminalHandle}.`, @@ -64,7 +92,9 @@ export function resolveOrchestrationCaller( params: OrchestrationCallerParams ): string | null { if (!params.evidenceAssertedByCaller) { - assertCallerHandleMatchesEvidence(runtime, params.callerTerminalHandle, params.callerEvidence) + assertCallerHandleMatchesEvidence(runtime, params.callerTerminalHandle, params.callerEvidence, { + callerAuthority: params.callerAuthority + }) } const paneKey = params.callerAuthority?.terminalHandle === params.callerTerminalHandle @@ -97,10 +127,13 @@ export function resolveRunScope(runtime: OrcaRuntimeService, params: RunScopePar orchestrationSkillRecoveryData() ) } - assertCallerHandleMatchesEvidence(runtime, params.callerTerminalHandle, params.callerEvidence) if (explicit && params.legacyCoordinatorRunId === explicit.id) { + assertCallerHandleMatchesEvidence(runtime, params.callerTerminalHandle, params.callerEvidence, { + allowLegacyAuthority: true + }) return explicit } + assertCallerHandleMatchesEvidence(runtime, params.callerTerminalHandle, params.callerEvidence) const paneKey = params.callerPaneKey ?? runtime.getTerminalPaneKey(params.callerTerminalHandle) if (!paneKey) { throw new OrchestrationError( @@ -109,7 +142,10 @@ export function resolveRunScope(runtime: OrcaRuntimeService, params: RunScopePar ) } const current = db.getCurrentRunForPane(paneKey) - if (!current) { + if ( + !current || + !isCallerCurrentRunCoordinator(runtime, current, params.callerTerminalHandle, paneKey) + ) { if (explicit) { throw new OrchestrationError( 'consumer_fenced', diff --git a/src/main/runtime/rpc/methods/orchestration-runs.test.ts b/src/main/runtime/rpc/methods/orchestration-runs.test.ts index 4afc18f4b242..8f0ddfbaa736 100644 --- a/src/main/runtime/rpc/methods/orchestration-runs.test.ts +++ b/src/main/runtime/rpc/methods/orchestration-runs.test.ts @@ -88,7 +88,11 @@ describe('orchestration RPC methods', () => { it('requires runtime-observed stable pane identity for binding', async () => { setup(false) - vi.spyOn(runtime, 'getTerminalPaneKey').mockReturnValue(null) + const transientPaneKey = 'tab_stale:33333333-3333-4333-8333-333333333333' + vi.spyOn(runtime, 'getTerminalPaneKey') + .mockReturnValueOnce(transientPaneKey) + .mockReturnValueOnce(transientPaneKey) + .mockReturnValue(null) await expect( call('orchestration.runCreate', { objective: 'No pane', from: 'term_stale' }) @@ -96,29 +100,76 @@ describe('orchestration RPC methods', () => { expect(db.listRuns().runs.filter((run) => run.legacy === 0)).toHaveLength(0) }) - it('rebinds explicitly, lists Runs, and keeps the legacy Run inspect-only', async () => { + it('refuses to replace a live coordinator and keeps the legacy Run inspect-only', async () => { setup(false) vi.spyOn(runtime, 'getTerminalPaneKey').mockImplementation((handle) => handle === 'term_old' ? 'tab_old:11111111-1111-4111-8111-111111111111' : 'tab_new:22222222-2222-4222-9222-222222222222' ) + vi.spyOn(runtime, 'getOrchestrationDispatchAuthority').mockImplementation((handle) => ({ + runtimeId: 'runtime_test', + terminalHandle: handle, + ptyId: handle, + worktreeId: 'folder:workspace', + processIncarnation: `runtime_test:${handle}:1`, + paneKey: + handle === 'term_old' + ? 'tab_old:11111111-1111-4111-8111-111111111111' + : 'tab_new:22222222-2222-4222-9222-222222222222', + launchTokenHash: null, + hostScope: { kind: 'local', hostId: 'local' } + })) const created = (await call('orchestration.runCreate', { objective: 'Move me', from: 'term_old' })) as { run: { id: string } } - const rebound = (await call('orchestration.runUse', { + vi.spyOn(runtime, 'inspectTerminalProcessIncarnationLiveness').mockResolvedValue('live') + await expect( + call('orchestration.runUse', { + id: created.run.id, + from: 'term_new' + }) + ).rejects.toMatchObject({ + code: 'consumer_fenced', + data: { + effectsApplied: false, + coordinatorStatus: 'live', + inspectCommandArgs: ['orchestration', 'run-show', '--id', created.run.id, '--json'], + retryCommandArgs: ['orchestration', 'run-use', '--id', created.run.id, '--json'], + nextSteps: [ + expect.stringContaining('owning coordinator terminal'), + expect.stringContaining('stop or exit that coordinator process'), + expect.stringContaining(`run-show --id ${created.run.id} --json`), + expect.stringContaining('No force-steal exists for an ordinary Run') + ] + } + }) + const inspected = (await call('orchestration.taskList', { + run: created.run.id, + callerTerminalHandle: 'term_new' + })) as { binding: { currentConsumer: boolean } } + const shown = (await call('orchestration.runShow', { id: created.run.id, from: 'term_new' - })) as { run: { consumer_generation: number } } + })) as { binding: { currentConsumer: boolean } } + const headlessShown = (await call('orchestration.runShow', { + id: created.run.id + })) as { binding: { currentConsumer: boolean } } + const headlessTasks = (await call('orchestration.taskList', { + run: created.run.id + })) as { binding: { currentConsumer: boolean } } const listed = (await call('orchestration.runList', {})) as { - runs: { id: string; legacy: number }[] + runs: { id: string; legacy: number; consumer_generation: number }[] } - expect(rebound.run.consumer_generation).toBe(2) + expect(inspected.binding.currentConsumer).toBe(false) + expect(shown.binding.currentConsumer).toBe(false) + expect(headlessShown.binding.currentConsumer).toBe(false) + expect(headlessTasks.binding.currentConsumer).toBe(false) expect(listed.runs).toEqual( expect.arrayContaining([ - expect.objectContaining({ id: created.run.id, legacy: 0 }), + expect.objectContaining({ id: created.run.id, legacy: 0, consumer_generation: 1 }), expect.objectContaining({ id: 'run_legacy_local', legacy: 1 }) ]) ) @@ -127,6 +178,486 @@ describe('orchestration RPC methods', () => { ).rejects.toMatchObject({ code: 'run_not_found' }) }) + it('refuses replacement when a migrated incumbent is not provably exited', async () => { + setup(false) + const oldPane = 'tab_old:11111111-1111-4111-8111-111111111111' + vi.spyOn(runtime, 'getTerminalPaneKey').mockReturnValue(oldPane) + vi.spyOn(runtime, 'getLiveTerminalPaneKey').mockImplementation((handle) => + handle === 'term_new' ? oldPane : null + ) + const run = db.createRun({ + objective: 'Migrated disconnected owner', + coordinatorHandle: 'term_old', + coordinatorPaneKey: oldPane + }) + db.db.prepare('UPDATE runs SET coordinator_authority_revision = -1 WHERE id = ?').run(run.id) + + await expect( + call('orchestration.taskCreate', { + spec: 'replacement write', + run: run.id, + callerTerminalHandle: 'term_new' + }) + ).rejects.toMatchObject({ code: 'consumer_fenced' }) + await expect( + call('orchestration.runUse', { id: run.id, from: 'term_new' }) + ).rejects.toMatchObject({ + code: 'consumer_fenced', + data: { + effectsApplied: false, + coordinatorStatus: 'unverifiable', + inspectCommandArgs: ['orchestration', 'run-show', '--id', run.id, '--json'], + retryCommandArgs: ['orchestration', 'run-use', '--id', run.id, '--json'], + nextSteps: expect.arrayContaining([ + expect.stringContaining('Restore connectivity to the owning host'), + expect.stringContaining('Loss of contact is not evidence of exit'), + expect.stringContaining(`run-show --id ${run.id} --json`), + expect.stringContaining('only after the owning host proves the incumbent exited') + ]) + } + }) + }) + + it('does not grant a migrated binding to a replacement incarnation on the same handle', async () => { + setup(false) + const pane = 'tab_old:11111111-1111-4111-8111-111111111111' + vi.spyOn(runtime, 'getTerminalPaneKey').mockReturnValue(pane) + vi.spyOn(runtime, 'getLiveTerminalPaneKey').mockReturnValue(pane) + vi.spyOn(runtime, 'getOrchestrationDispatchAuthority').mockReturnValue({ + runtimeId: 'runtime_test', + terminalHandle: 'term_old', + ptyId: 'pty_replacement', + worktreeId: 'folder:workspace', + processIncarnation: 'pty_replacement:incarnation-2', + paneKey: pane, + launchTokenHash: 'replacement-launch', + hostScope: { kind: 'local', hostId: 'local' } + }) + const run = db.createRun({ + objective: 'Migrated same-handle owner', + coordinatorHandle: 'term_old', + coordinatorPaneKey: pane + }) + db.db.prepare('UPDATE runs SET coordinator_authority_revision = -1 WHERE id = ?').run(run.id) + + await expect( + call('orchestration.runUse', { id: run.id, from: 'term_old' }) + ).rejects.toMatchObject({ + code: 'consumer_fenced', + data: { effectsApplied: false, coordinatorStatus: 'live' } + }) + expect(db.getRun(run.id)).toMatchObject({ + coordinator_handle: 'term_old', + coordinator_process_incarnation: null, + consumer_generation: run.consumer_generation + }) + }) + + it('backfills an exact migrated handle only from restored process authority', async () => { + setup(false) + const pane = 'tab_old:11111111-1111-4111-8111-111111111111' + const processIncarnation = 'pty_retained:incarnation-1' + vi.spyOn(runtime, 'getTerminalPaneKey').mockReturnValue(pane) + vi.spyOn(runtime, 'getOrchestrationDispatchAuthority').mockReturnValue({ + runtimeId: 'runtime_test', + terminalHandle: 'term_old', + ptyId: 'pty_retained', + worktreeId: 'folder:workspace', + processIncarnation, + paneKey: pane, + launchTokenHash: null, + hostScope: { kind: 'local', hostId: 'local' } + }) + ctx = { + runtime, + orchestrationCompatibilityEvidence: { + terminalHandle: 'term_old', + paneKey: pane, + launchToken: 'retained-launch' + } + } + let attestedProcessIncarnation = 'pty_retained:incarnation-stale' + let verificationCount = 0 + let dropAttestationAfterFirstVerification = false + vi.spyOn(runtime, 'verifyOrchestrationCompatibilityCaller').mockImplementation(() => { + verificationCount += 1 + return dropAttestationAfterFirstVerification && verificationCount > 1 + ? null + : { + terminalHandle: 'term_old', + paneKey: pane, + processIncarnation: attestedProcessIncarnation, + launchTokenHash: 'retained-launch-hash', + hostScope: { kind: 'local', hostId: 'local' }, + terminalProvenance: 'restored' + } + }) + const run = db.createRun({ + objective: 'Migrated retained owner', + coordinatorHandle: 'term_old', + coordinatorPaneKey: pane + }) + db.db.prepare('UPDATE runs SET coordinator_authority_revision = -1 WHERE id = ?').run(run.id) + + await expect( + call('orchestration.runUse', { id: run.id, from: 'term_old' }) + ).rejects.toMatchObject({ code: 'consumer_fenced' }) + attestedProcessIncarnation = processIncarnation + verificationCount = 0 + dropAttestationAfterFirstVerification = true + + await expect( + call('orchestration.runUse', { id: run.id, from: 'term_old' }) + ).rejects.toMatchObject({ code: 'consumer_fenced' }) + verificationCount = 0 + dropAttestationAfterFirstVerification = false + + const rebound = (await call('orchestration.runUse', { + id: run.id, + from: 'term_old' + })) as { run: { consumer_generation: number; coordinator_process_incarnation: string } } + + expect(rebound.run).toMatchObject({ + consumer_generation: run.consumer_generation, + coordinator_process_incarnation: processIncarnation + }) + }) + + it('recovers a migrated binding after the resolved incumbent process is proven exited', async () => { + setup(false) + const oldPane = 'tab_old:11111111-1111-4111-8111-111111111111' + const newPane = 'tab_new:22222222-2222-4222-9222-222222222222' + vi.spyOn(runtime, 'getTerminalPaneKey').mockImplementation((handle) => + handle === 'term_old' ? oldPane : newPane + ) + vi.spyOn(runtime, 'getOrchestrationDispatchAuthority').mockImplementation((handle) => ({ + runtimeId: 'runtime_test', + terminalHandle: handle, + ptyId: handle, + worktreeId: 'folder:workspace', + processIncarnation: `${handle}:incarnation-1`, + paneKey: handle === 'term_old' ? oldPane : newPane, + launchTokenHash: null, + hostScope: { kind: 'local', hostId: 'local' } + })) + const run = db.createRun({ + objective: 'Migrated exited owner', + coordinatorHandle: 'term_old', + coordinatorPaneKey: oldPane + }) + db.db.prepare('UPDATE runs SET coordinator_authority_revision = -1 WHERE id = ?').run(run.id) + vi.spyOn(runtime, 'inspectTerminalProcessIncarnationLiveness').mockResolvedValue('exited') + + const rebound = (await call('orchestration.runUse', { + id: run.id, + from: 'term_new' + })) as { run: { coordinator_handle: string; consumer_generation: number } } + + expect(rebound.run).toMatchObject({ + coordinator_handle: 'term_new', + consumer_generation: run.consumer_generation + 1 + }) + }) + + it('backfills a migrated binding when a reminted handle resolves to the same process', async () => { + setup(false) + const oldPane = 'tab_old:11111111-1111-4111-8111-111111111111' + const remintedPane = 'tab_new:11111111-1111-4111-8111-111111111111' + vi.spyOn(runtime, 'getTerminalPaneKey').mockImplementation((handle) => + handle === 'term_old' ? oldPane : remintedPane + ) + vi.spyOn(runtime, 'getOrchestrationDispatchAuthority').mockImplementation((handle) => ({ + runtimeId: 'runtime_test', + terminalHandle: handle, + ptyId: 'pty_coord', + worktreeId: 'folder:workspace', + processIncarnation: 'pty_coord:incarnation-1', + paneKey: handle === 'term_old' ? oldPane : remintedPane, + launchTokenHash: null, + hostScope: { kind: 'local', hostId: 'local' } + })) + const run = db.createRun({ + objective: 'Migrated live owner', + coordinatorHandle: 'term_old', + coordinatorPaneKey: oldPane + }) + + const rebound = (await call('orchestration.runUse', { + id: run.id, + from: 'term_reminted' + })) as { run: { coordinator_handle: string; consumer_generation: number } } + + expect(rebound.run).toMatchObject({ + coordinator_handle: 'term_reminted', + consumer_generation: run.consumer_generation + }) + }) + + it('preserves the consumer generation when the same process remints its handle and pane', async () => { + setup(false) + const oldPane = 'tab_old:11111111-1111-4111-8111-111111111111' + const remintedPane = 'tab_new:11111111-1111-4111-8111-111111111111' + vi.spyOn(runtime, 'getTerminalPaneKey').mockImplementation((handle) => + handle === 'term_old' ? oldPane : remintedPane + ) + vi.spyOn(runtime, 'getOrchestrationDispatchAuthority').mockImplementation((handle) => ({ + runtimeId: 'runtime_test', + terminalHandle: handle, + ptyId: 'pty_coord', + worktreeId: 'folder:workspace', + processIncarnation: 'pty_coord:incarnation-1', + paneKey: handle === 'term_old' ? oldPane : remintedPane, + launchTokenHash: null, + hostScope: { kind: 'local', hostId: 'local' } + })) + const created = (await call('orchestration.runCreate', { + objective: 'Survive remint', + from: 'term_old' + })) as { run: { id: string; consumer_generation: number } } + + const rebound = (await call('orchestration.runUse', { + id: created.run.id, + from: 'term_reminted' + })) as { run: { coordinator_handle: string; consumer_generation: number } } + + expect(rebound.run).toMatchObject({ + coordinator_handle: 'term_reminted', + consumer_generation: created.run.consumer_generation + }) + }) + + it('does not grant Run authority to a replacement process in the same pane', async () => { + setup(false) + const pane = 'tab_coord:11111111-1111-4111-8111-111111111111' + vi.spyOn(runtime, 'getTerminalPaneKey').mockReturnValue(pane) + vi.spyOn(runtime, 'getOrchestrationDispatchAuthority').mockImplementation((handle) => ({ + runtimeId: 'runtime_test', + terminalHandle: handle, + ptyId: handle, + worktreeId: 'folder:workspace', + processIncarnation: `${handle}:incarnation-1`, + paneKey: pane, + launchTokenHash: null, + hostScope: { kind: 'local', hostId: 'local' } + })) + const created = (await call('orchestration.runCreate', { + objective: 'Keep incumbent authority', + from: 'term_old' + })) as { run: { id: string } } + + await expect( + call('orchestration.taskCreate', { + spec: 'replacement write', + run: created.run.id, + callerTerminalHandle: 'term_new' + }) + ).rejects.toMatchObject({ code: 'consumer_fenced' }) + + vi.spyOn(runtime, 'inspectTerminalProcessIncarnationLiveness').mockResolvedValue('live') + await expect( + call('orchestration.runUse', { id: created.run.id, from: 'term_new' }) + ).rejects.toMatchObject({ + code: 'consumer_fenced', + data: { effectsApplied: false, coordinatorStatus: 'live' } + }) + + await expect( + call('orchestration.runCreate', { + objective: 'Must not evict the live owner', + from: 'term_new' + }) + ).rejects.toMatchObject({ code: 'consumer_fenced', data: { effectsApplied: false } }) + expect(db.getRun(created.run.id)).toMatchObject({ + coordinator_handle: 'term_old', + consumer_generation: 1 + }) + + const unboundTarget = db.createRun({ + objective: 'Unbound target', + coordinatorHandle: 'term_target', + coordinatorPaneKey: 'tab_target:33333333-3333-4333-8333-333333333333' + }) + db.db + .prepare( + `UPDATE runs + SET coordinator_handle = NULL, coordinator_pane_key = NULL, + coordinator_process_incarnation = NULL, coordinator_host_scope = NULL + WHERE id = ?` + ) + .run(unboundTarget.id) + await expect( + call('orchestration.runUse', { id: unboundTarget.id, from: 'term_new' }) + ).rejects.toMatchObject({ code: 'consumer_fenced', data: { effectsApplied: false } }) + expect(db.getRun(created.run.id)).toMatchObject({ + coordinator_handle: 'term_old', + consumer_generation: 1 + }) + }) + + it('allows a distinct coordinator only after the owning host proves the incumbent exited', async () => { + setup(false) + const oldPane = 'tab_old:11111111-1111-4111-8111-111111111111' + const newPane = 'tab_new:22222222-2222-4222-9222-222222222222' + vi.spyOn(runtime, 'getTerminalPaneKey').mockImplementation((handle) => + handle === 'term_old' ? oldPane : newPane + ) + vi.spyOn(runtime, 'getOrchestrationDispatchAuthority').mockImplementation((handle) => ({ + runtimeId: 'runtime_test', + terminalHandle: handle, + ptyId: handle, + worktreeId: 'folder:workspace', + processIncarnation: `runtime_test:${handle}:1`, + paneKey: handle === 'term_old' ? oldPane : newPane, + launchTokenHash: null, + hostScope: { kind: 'local', hostId: 'local' } + })) + const created = (await call('orchestration.runCreate', { + objective: 'Recover exited owner', + from: 'term_old' + })) as { run: { id: string; consumer_generation: number } } + vi.spyOn(runtime, 'inspectTerminalProcessIncarnationLiveness').mockResolvedValue('exited') + + const rebound = (await call('orchestration.runUse', { + id: created.run.id, + from: 'term_new' + })) as { run: { coordinator_handle: string; consumer_generation: number } } + + expect(rebound.run).toMatchObject({ + coordinator_handle: 'term_new', + consumer_generation: created.run.consumer_generation + 1 + }) + }) + + it('binds an authority-less migrated Run without requiring a nonexistent incumbent', async () => { + setup(false) + const pane = 'tab_new:22222222-2222-4222-9222-222222222222' + vi.spyOn(runtime, 'getTerminalPaneKey').mockReturnValue(pane) + vi.spyOn(runtime, 'getOrchestrationDispatchAuthority').mockImplementation((handle) => ({ + runtimeId: 'runtime_test', + terminalHandle: handle, + ptyId: handle, + worktreeId: 'folder:workspace', + processIncarnation: `${handle}:incarnation-1`, + paneKey: pane, + launchTokenHash: null, + hostScope: { kind: 'local', hostId: 'local' } + })) + const run = db.createRun({ + objective: 'Previously unbound', + coordinatorHandle: 'term_old', + coordinatorPaneKey: 'tab_old:11111111-1111-4111-8111-111111111111' + }) + db.db + .prepare( + `UPDATE runs + SET coordinator_handle = NULL, coordinator_pane_key = NULL, + coordinator_process_incarnation = NULL, coordinator_host_scope = NULL + WHERE id = ?` + ) + .run(run.id) + + const rebound = (await call('orchestration.runUse', { + id: run.id, + from: 'term_new' + })) as { run: { coordinator_handle: string } } + + expect(rebound.run.coordinator_handle).toBe('term_new') + }) + + it('cancels the Run bound during an asynchronous incumbent observation', async () => { + setup(false) + const oldPane = 'tab_old:11111111-1111-4111-8111-111111111111' + const newPane = 'tab_new:22222222-2222-4222-9222-222222222222' + vi.spyOn(runtime, 'getTerminalPaneKey').mockImplementation((handle) => + handle === 'term_old' ? oldPane : newPane + ) + vi.spyOn(runtime, 'getOrchestrationDispatchAuthority').mockImplementation((handle) => ({ + runtimeId: 'runtime_test', + terminalHandle: handle, + ptyId: handle, + worktreeId: 'folder:workspace', + processIncarnation: `${handle}:incarnation-1`, + paneKey: handle === 'term_old' ? oldPane : newPane, + launchTokenHash: null, + hostScope: { kind: 'local', hostId: 'local' } + })) + const target = (await call('orchestration.runCreate', { + objective: 'Target Run', + from: 'term_old' + })) as { run: { id: string } } + let finishObservation: ((status: 'exited') => void) | undefined + vi.spyOn(runtime, 'inspectTerminalProcessIncarnationLiveness').mockImplementation( + () => new Promise((resolve) => (finishObservation = resolve)) + ) + const cancel = vi.spyOn(runtime, 'cancelMessageWaiters') + + const use = call('orchestration.runUse', { id: target.run.id, from: 'term_new' }) + await vi.waitFor(() => expect(finishObservation).toBeTypeOf('function')) + const overlapping = db.createRun({ + objective: 'Overlapping Run', + coordinatorHandle: 'term_new', + coordinatorPaneKey: newPane, + coordinatorProcessIncarnation: 'term_new:incarnation-1', + coordinatorHostScope: JSON.stringify({ kind: 'local', hostId: 'local' }) + }) + finishObservation?.('exited') + await use + + expect(cancel).toHaveBeenCalledWith(`run:${overlapping.id}`) + }) + + it('rejects a claimant process that reincarnates during incumbent observation', async () => { + setup(false) + const oldPane = 'tab_old:11111111-1111-4111-8111-111111111111' + const newPane = 'tab_new:22222222-2222-4222-9222-222222222222' + let claimantIncarnation = 'term_new:incarnation-1' + vi.spyOn(runtime, 'getTerminalPaneKey').mockImplementation((handle) => + handle === 'term_old' ? oldPane : newPane + ) + vi.spyOn(runtime, 'getOrchestrationDispatchAuthority').mockImplementation((handle) => ({ + runtimeId: 'runtime_test', + terminalHandle: handle, + ptyId: handle, + worktreeId: 'folder:workspace', + processIncarnation: handle === 'term_old' ? 'term_old:incarnation-1' : claimantIncarnation, + paneKey: handle === 'term_old' ? oldPane : newPane, + launchTokenHash: null, + hostScope: { kind: 'local', hostId: 'local' } + })) + const target = (await call('orchestration.runCreate', { + objective: 'Keep claimant exact', + from: 'term_old' + })) as { run: { id: string } } + let finishObservation: ((status: 'exited') => void) | undefined + vi.spyOn(runtime, 'inspectTerminalProcessIncarnationLiveness').mockImplementation( + () => new Promise((resolve) => (finishObservation = resolve)) + ) + + const use = call('orchestration.runUse', { id: target.run.id, from: 'term_new' }) + await vi.waitFor(() => expect(finishObservation).toBeTypeOf('function')) + claimantIncarnation = 'term_new:incarnation-2' + finishObservation?.('exited') + + await expect(use).rejects.toMatchObject({ + code: 'consumer_fenced', + data: { + effectsApplied: false, + coordinatorStatus: 'unverifiable', + claimantStatus: 'changed', + inspectCommandArgs: ['orchestration', 'run-show', '--id', target.run.id, '--json'], + retryCommandArgs: ['orchestration', 'run-use', '--id', target.run.id, '--json'], + nextSteps: expect.arrayContaining([ + expect.stringContaining('same Orca CLI executable'), + expect.stringContaining('one stable replacement agent process') + ]) + } + }) + expect(db.getRun(target.run.id)).toMatchObject({ + coordinator_handle: 'term_old', + consumer_generation: 1 + }) + }) + it('requires an explicit binding before task mutation', async () => { setup(false) vi.spyOn(runtime, 'getTerminalPaneKey').mockReturnValue(coordinatorPaneKey) @@ -150,6 +681,7 @@ describe('orchestration RPC methods', () => { setup(false) const oldPane = 'tab_old:11111111-1111-4111-8111-111111111111' const newPane = 'tab_new:22222222-2222-4222-9222-222222222222' + const otherPane = 'tab_other:33333333-3333-4333-8333-333333333333' vi.spyOn(runtime, 'getTerminalPaneKey').mockImplementation((handle) => handle === 'term_old' ? oldPane : newPane ) @@ -161,7 +693,7 @@ describe('orchestration RPC methods', () => { const runB = db.createRun({ objective: 'B', coordinatorHandle: 'term_other', - coordinatorPaneKey: newPane + coordinatorPaneKey: otherPane }) const taskA = db.createTask({ spec: 'A work', runId: runA.id }) db.createTask({ spec: 'B work', runId: runB.id }) @@ -174,7 +706,14 @@ describe('orchestration RPC methods', () => { db.bindRun({ runId: runA.id, coordinatorHandle: 'term_new', - coordinatorPaneKey: newPane + coordinatorPaneKey: newPane, + incumbentObservation: { + coordinatorHandle: 'term_old', + coordinatorPaneKey: oldPane, + coordinatorProcessIncarnation: null, + coordinatorHostScope: null, + status: 'exited' + } }) await expect( call('orchestration.taskCreate', { @@ -192,6 +731,16 @@ describe('orchestration RPC methods', () => { vi.spyOn(runtime, 'getTerminalPaneKey').mockImplementation((handle) => handle === 'term_old' ? oldPane : newPane ) + vi.spyOn(runtime, 'getOrchestrationDispatchAuthority').mockImplementation((handle) => ({ + runtimeId: 'runtime_test', + terminalHandle: handle, + ptyId: handle, + worktreeId: 'folder:workspace', + processIncarnation: `runtime_test:${handle}:1`, + paneKey: handle === 'term_old' ? oldPane : newPane, + launchTokenHash: null, + hostScope: { kind: 'local', hostId: 'local' } + })) const created = (await call('orchestration.runCreate', { objective: 'Wait fencing', from: 'term_old' @@ -203,6 +752,7 @@ describe('orchestration RPC methods', () => { }) const fenced = expect(oldWait).rejects.toMatchObject({ code: 'consumer_fenced' }) await Promise.resolve() + vi.spyOn(runtime, 'inspectTerminalProcessIncarnationLiveness').mockResolvedValue('exited') await call('orchestration.runUse', { id: created.run.id, diff --git a/src/main/runtime/rpc/methods/orchestration-runs.ts b/src/main/runtime/rpc/methods/orchestration-runs.ts index 7938bc240f19..9030ec5366e4 100644 --- a/src/main/runtime/rpc/methods/orchestration-runs.ts +++ b/src/main/runtime/rpc/methods/orchestration-runs.ts @@ -7,6 +7,14 @@ import { assertCallerHandleMatchesEvidence, resolveOrchestrationCaller } from './orchestration-run-scope' +import { isCurrentRunCoordinator } from '../../orchestration/run-coordinator-authority' +import { isEquivalentPaneKey } from '../../orchestration/db/pane-key-match' +import { + isCallerCurrentRunCoordinator, + resolveAttestedRunCoordinatorPane, + resolveRunCoordinatorIdentity +} from './orchestration-coordinator-caller' +import { observeRunCoordinator } from './orchestration-run-coordinator-observation' const RunCreateParams = z.object({ objective: requiredString('Missing --objective'), @@ -38,10 +46,13 @@ export const ORCHESTRATION_RUN_METHODS: RpcMethod[] = [ }) const db = runtime.getOrchestrationDb() const priorRun = db.getCurrentRunForPane(paneKey) + const identity = resolveRunCoordinatorIdentity(runtime, params.from, paneKey) const run = db.createRun({ objective: params.objective, coordinatorHandle: params.from, - coordinatorPaneKey: paneKey + coordinatorPaneKey: paneKey, + coordinatorProcessIncarnation: identity.processIncarnation, + coordinatorHostScope: identity.hostScope }) runtime.cancelMessageWaiters(params.from) if (priorRun) { @@ -53,15 +64,19 @@ export const ORCHESTRATION_RUN_METHODS: RpcMethod[] = [ defineMethod({ name: 'orchestration.runUse', params: RunUseParams, - handler: ( + handler: async ( params, { runtime, legacyCoordinatorAuthority, orchestrationCompatibilityEvidence, - orchestrationCompatibilityCallerAuthority: callerAuthority + orchestrationCompatibilityCallerAuthority: preflightCallerAuthority } ) => { + let callerAuthority = + preflightCallerAuthority ?? + runtime.verifyOrchestrationCompatibilityCaller(orchestrationCompatibilityEvidence) ?? + undefined const paneKey = resolveOrchestrationCaller(runtime, { callerTerminalHandle: params.from, callerEvidence: orchestrationCompatibilityEvidence, @@ -79,13 +94,131 @@ export const ORCHESTRATION_RUN_METHODS: RpcMethod[] = [ { effectsApplied: false } ) } - assertCallerHandleMatchesEvidence(runtime, params.from, orchestrationCompatibilityEvidence) const db = runtime.getOrchestrationDb() + const targetRun = db.getRun(params.id) + const identity = resolveRunCoordinatorIdentity(runtime, params.from, paneKey) + const incumbentIdentity = + targetRun && + !targetRun.coordinator_process_incarnation && + targetRun.coordinator_handle && + targetRun.coordinator_handle !== params.from + ? resolveRunCoordinatorIdentity( + runtime, + targetRun.coordinator_handle, + targetRun.coordinator_pane_key + ) + : null + const dynamicProcessContinuity = Boolean( + incumbentIdentity?.processIncarnation && + identity.processIncarnation && + incumbentIdentity.processIncarnation === identity.processIncarnation && + incumbentIdentity.hostScope === identity.hostScope + ) + const persistedProcessContinuity = Boolean( + targetRun?.coordinator_process_incarnation && + targetRun.coordinator_handle !== params.from && + targetRun.coordinator_process_incarnation === identity.processIncarnation && + targetRun.coordinator_host_scope === identity.hostScope + ) + const sameProcessRemint = dynamicProcessContinuity || persistedProcessContinuity + if (!callerAuthority && sameProcessRemint && orchestrationCompatibilityEvidence) { + callerAuthority = + runtime.verifyOrchestrationCompatibilityCaller(orchestrationCompatibilityEvidence, { + currentRuntimeLaunchSufficient: true + }) ?? undefined + } + assertCallerHandleMatchesEvidence(runtime, params.from, orchestrationCompatibilityEvidence, { + callerAuthority, + allowLegacyAuthority: Boolean(legacyCoordinatorAuthority) + }) + const restoredMigratedContinuity = Boolean( + targetRun && + targetRun.coordinator_authority_revision < 0 && + callerAuthority?.terminalProvenance === 'restored' && + callerAuthority.processIncarnation === identity.processIncarnation && + JSON.stringify(callerAuthority.hostScope) === identity.hostScope && + targetRun.coordinator_handle === params.from && + targetRun.coordinator_pane_key && + isEquivalentPaneKey(targetRun.coordinator_pane_key, paneKey) + ) + const sameAuthority = targetRun + ? isCurrentRunCoordinator(targetRun, identity) || + dynamicProcessContinuity || + restoredMigratedContinuity + : false + const incumbentObservation = + targetRun && !sameAuthority + ? await observeRunCoordinator(runtime, targetRun, incumbentIdentity) + : undefined + const currentPaneKey = resolveOrchestrationCaller(runtime, { + callerTerminalHandle: params.from, + callerEvidence: orchestrationCompatibilityEvidence, + callerAuthority, + requireStablePane: true, + evidenceAssertedByCaller: true + }) + assertCallerHandleMatchesEvidence(runtime, params.from, orchestrationCompatibilityEvidence, { + callerAuthority, + allowLegacyAuthority: Boolean(legacyCoordinatorAuthority) + }) + const currentIdentity = resolveRunCoordinatorIdentity(runtime, params.from, currentPaneKey) + const revalidatedCaller = orchestrationCompatibilityEvidence + ? runtime.verifyOrchestrationCompatibilityCaller( + orchestrationCompatibilityEvidence, + params.takeoverLegacy || sameProcessRemint + ? { currentRuntimeLaunchSufficient: true } + : undefined + ) + : null + const restoredContinuityStillAttested = + !restoredMigratedContinuity || + Boolean( + revalidatedCaller?.terminalProvenance === 'restored' && + revalidatedCaller.terminalHandle === params.from && + revalidatedCaller.paneKey === paneKey && + revalidatedCaller.processIncarnation === currentIdentity.processIncarnation && + JSON.stringify(revalidatedCaller.hostScope) === currentIdentity.hostScope + ) + const claimantStillLive = revalidatedCaller + ? revalidatedCaller.terminalHandle === params.from && revalidatedCaller.paneKey === paneKey + : legacyCoordinatorAuthority || !orchestrationCompatibilityEvidence + ? runtime.getLiveTerminalPaneKey(params.from) === paneKey + : false + if ( + (incumbentObservation && !claimantStillLive) || + !restoredContinuityStillAttested || + currentPaneKey !== paneKey || + currentIdentity.processIncarnation !== identity.processIncarnation || + currentIdentity.hostScope !== identity.hostScope + ) { + const coordinatorStatus = incumbentObservation?.status === 'live' ? 'live' : 'unverifiable' + throw new OrchestrationError( + 'consumer_fenced', + 'The claiming coordinator process changed while Run authority was being checked. No effects were applied.', + { + effectsApplied: false, + coordinatorStatus, + claimantStatus: 'changed', + inspectCommandArgs: ['orchestration', 'run-show', '--id', params.id, '--json'], + retryCommandArgs: ['orchestration', 'run-use', '--id', params.id, '--json'], + nextSteps: [ + `Inspect current authority by running orchestration run-show --id ${params.id} --json with the same Orca CLI executable.`, + coordinatorStatus === 'live' + ? 'Continue from the owning coordinator; do not retry while it remains live.' + : `From one stable replacement agent process, run orchestration run-use --id ${params.id} --json with that executable so Orca can re-prove the incumbent state.` + ] + } + ) + } const priorRun = db.getCurrentRunForPane(paneKey) const run = db.bindRun({ runId: params.id, coordinatorHandle: params.from, coordinatorPaneKey: paneKey, + coordinatorProcessIncarnation: identity.processIncarnation, + coordinatorHostScope: identity.hostScope, + authorityContinuity: sameProcessRemint || restoredMigratedContinuity, + incumbentObservation, takeoverLegacy: params.takeoverLegacy, legacyCoordinatorAuthority }) @@ -112,7 +245,10 @@ export const ORCHESTRATION_RUN_METHODS: RpcMethod[] = [ callerEvidence: orchestrationCompatibilityEvidence, requireStablePane: true }) - return { run: runtime.getOrchestrationDb().getCurrentRunForPane(paneKey) ?? null } + const run = runtime.getOrchestrationDb().getCurrentRunForPane(paneKey) + return { + run: run && isCallerCurrentRunCoordinator(runtime, run, params.from, paneKey) ? run : null + } } }), defineMethod({ @@ -123,12 +259,31 @@ export const ORCHESTRATION_RUN_METHODS: RpcMethod[] = [ defineMethod({ name: 'orchestration.runShow', params: RunShowParams, - handler: (params, { runtime }) => { - const run = runtime.getOrchestrationDb().getRun(params.id) + handler: (params, { orchestrationCompatibilityEvidence, runtime }) => { + const db = runtime.getOrchestrationDb() + const run = db.getRun(params.id) if (!run) { throw new OrchestrationError('run_not_found', `Run ${params.id} was not found.`) } - return { run } + const callerPaneKey = params.from + ? resolveAttestedRunCoordinatorPane( + runtime, + run, + params.from, + orchestrationCompatibilityEvidence + ) + : null + return { + run, + binding: { + currentConsumer: Boolean( + params.from && + run.legacy === 0 && + callerPaneKey !== null && + db.getCurrentRunForPane(callerPaneKey)?.id === run.id + ) + } + } } }) ] diff --git a/src/main/runtime/rpc/methods/orchestration-send.test.ts b/src/main/runtime/rpc/methods/orchestration-send.test.ts index e4312c34a3a1..e6578ac6c0e6 100644 --- a/src/main/runtime/rpc/methods/orchestration-send.test.ts +++ b/src/main/runtime/rpc/methods/orchestration-send.test.ts @@ -76,7 +76,8 @@ describe('orchestration RPC methods', () => { db.bindRun({ runId: activeRunId!, coordinatorHandle: 'term_reminted', - coordinatorPaneKey: remintedPaneKey + coordinatorPaneKey: remintedPaneKey, + authorityContinuity: true }) const waiting = runtime.waitForMessage(`run:${activeRunId}`, { timeoutMs: 5_000 }) diff --git a/src/main/runtime/rpc/methods/orchestration-tasks-dispatch.test.ts b/src/main/runtime/rpc/methods/orchestration-tasks-dispatch.test.ts index 4a87c53059d3..cefca7e074fc 100644 --- a/src/main/runtime/rpc/methods/orchestration-tasks-dispatch.test.ts +++ b/src/main/runtime/rpc/methods/orchestration-tasks-dispatch.test.ts @@ -53,7 +53,7 @@ describe('orchestration RPC methods', () => { }) it('records the caller pane, process, and Run generation when creating a task', async () => { - setup() + setup(false) vi.spyOn(runtime, 'getTerminalPaneKey').mockImplementation((handle) => handle === 'term_creator' ? coordinatorPaneKey : null ) @@ -62,9 +62,14 @@ describe('orchestration RPC methods', () => { paneKey: coordinatorPaneKey, processIncarnation: 'pty-creator:incarnation-a' } as never) + const created = (await call('orchestration.runCreate', { + objective: 'Creator authority', + from: 'term_creator' + })) as { run: { id: string } } const result = (await call('orchestration.taskCreate', { spec: 'spawn related workspace', - callerTerminalHandle: 'term_creator' + callerTerminalHandle: 'term_creator', + run: created.run.id })) as { task: { id: string } } expect(db.getTask(result.task.id)).toMatchObject({ diff --git a/src/main/runtime/rpc/methods/orchestration-worker-start-prompt-contract.test.ts b/src/main/runtime/rpc/methods/orchestration-worker-start-prompt-contract.test.ts index 826eb57ea559..3e48da0efbca 100644 --- a/src/main/runtime/rpc/methods/orchestration-worker-start-prompt-contract.test.ts +++ b/src/main/runtime/rpc/methods/orchestration-worker-start-prompt-contract.test.ts @@ -105,7 +105,23 @@ async function createPromptContractHarness( : null ) vi.spyOn(runtime, 'getTerminalProcessIncarnation').mockImplementation((candidate) => - candidate === handle ? `runtime_test:${handle}:1` : null + candidate === handle + ? `runtime_test:${handle}:1` + : candidate === 'term_coord' + ? 'runtime_test:term_coord:1' + : null + ) + vi.spyOn(runtime, 'verifyOrchestrationCompatibilityCaller').mockImplementation((evidence) => + evidence?.terminalHandle === 'term_coord' && evidence.paneKey === COORDINATOR_PANE_KEY + ? { + terminalHandle: evidence.terminalHandle, + paneKey: evidence.paneKey, + processIncarnation: 'runtime_test:term_coord:1', + hostScope: { kind: 'local', hostId: 'local' }, + launchTokenHash: 'test-token-hash', + terminalProvenance: 'current_runtime' + } + : null ) vi.spyOn(runtime, 'validateOrchestrationAgentLauncher').mockImplementation(() => {}) vi.spyOn(runtime, 'showTerminal').mockResolvedValue({ @@ -139,6 +155,11 @@ async function createPromptContractHarness( authToken: 'caller-token', orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION, orchestrationRequestId: `${REQUEST_ID}_${outcome}`, + orchestrationCompatibilityEvidence: { + terminalHandle: 'term_coord', + paneKey: COORDINATOR_PANE_KEY, + launchToken: 'coordinator-launch-token' + }, method: 'orchestration.workerStart', params: { task: task.id, diff --git a/src/main/runtime/rpc/methods/orchestration-workers-new-worktree.test.ts b/src/main/runtime/rpc/methods/orchestration-workers-new-worktree.test.ts index 2410a74c10f3..7714baf5490c 100644 --- a/src/main/runtime/rpc/methods/orchestration-workers-new-worktree.test.ts +++ b/src/main/runtime/rpc/methods/orchestration-workers-new-worktree.test.ts @@ -34,7 +34,23 @@ describe('orchestration new-worktree workers', () => { : null ) vi.spyOn(runtime, 'getTerminalProcessIncarnation').mockImplementation((handle) => - handle === 'term_worker' ? 'runtime_test:term_worker:1' : null + handle === 'term_worker' + ? 'runtime_test:term_worker:1' + : handle === 'term_coord' + ? 'runtime_test:term_coord:1' + : null + ) + vi.spyOn(runtime, 'verifyOrchestrationCompatibilityCaller').mockImplementation((evidence) => + evidence?.terminalHandle === 'term_coord' && evidence.paneKey === coordinatorPaneKey + ? { + terminalHandle: evidence.terminalHandle, + paneKey: evidence.paneKey, + processIncarnation: 'runtime_test:term_coord:1', + hostScope: { kind: 'local', hostId: 'local' }, + launchTokenHash: 'test-token-hash', + terminalProvenance: 'current_runtime' + } + : null ) vi.spyOn(runtime, 'validateOrchestrationAgentLauncher').mockImplementation(() => {}) vi.spyOn(runtime, 'showTerminal').mockResolvedValue({ @@ -537,6 +553,11 @@ describe('orchestration new-worktree workers', () => { authToken: 'caller-token', orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION, orchestrationRequestId: 'worker_start_request', + orchestrationCompatibilityEvidence: { + terminalHandle: 'term_coord', + paneKey: coordinatorPaneKey, + launchToken: 'coordinator-launch-token' + }, method: 'orchestration.workerStart', params: { task: task.id, @@ -604,6 +625,11 @@ describe('orchestration new-worktree workers', () => { authToken: 'caller-token', orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION, orchestrationRequestId: 'worker_start_request', + orchestrationCompatibilityEvidence: { + terminalHandle: 'term_coord', + paneKey: coordinatorPaneKey, + launchToken: 'coordinator-launch-token' + }, method: 'orchestration.workerStart', params: { task: task.id, @@ -632,6 +658,20 @@ describe('orchestration new-worktree workers', () => { ? `tab_coord_reminted:${coordinatorPaneKey.split(':')[1]}` : null ) + vi.spyOn(restartedRuntime, 'verifyOrchestrationCompatibilityCaller').mockImplementation( + (evidence) => + evidence?.terminalHandle === 'term_coord_reminted' && + evidence.paneKey === `tab_coord_reminted:${coordinatorPaneKey.split(':')[1]}` + ? { + terminalHandle: evidence.terminalHandle, + paneKey: evidence.paneKey, + processIncarnation: 'runtime_test:term_coord:1', + hostScope: { kind: 'local', hostId: 'local' }, + launchTokenHash: 'test-token-hash', + terminalProvenance: 'restored' + } + : null + ) const recreateWorktree = vi .spyOn(restartedRuntime, 'createManagedWorktree') .mockRejectedValue(new Error('replay recreated the worktree')) @@ -646,6 +686,11 @@ describe('orchestration new-worktree workers', () => { ...request, id: 'rpc_worker_start_retry', authToken: 'caller-token-after-restart', + orchestrationCompatibilityEvidence: { + terminalHandle: 'term_coord_reminted', + paneKey: `tab_coord_reminted:${coordinatorPaneKey.split(':')[1]}`, + launchToken: 'coordinator-launch-token-after-restart' + }, params: { ...(request.params as Record), from: 'term_coord_reminted' } }) diff --git a/src/main/runtime/rpc/methods/orchestration-workers.ts b/src/main/runtime/rpc/methods/orchestration-workers.ts index 612715259392..7a5a0cffc74c 100644 --- a/src/main/runtime/rpc/methods/orchestration-workers.ts +++ b/src/main/runtime/rpc/methods/orchestration-workers.ts @@ -1,3 +1,7 @@ +import { + isWorkerStartTimeoutWithinTimerLimit, + resolveWorkerStartReadinessTimeoutMs +} from '../../../../shared/orchestration-timing-budgets' import type { TuiAgent } from '../../../../shared/tui-agent' import { buildDispatchPreamble } from '../../orchestration/preamble' import { OrchestrationError } from '../../orchestration/orchestration-error' @@ -22,10 +26,7 @@ import { failWorkerStartWithReceipt } from './orchestration-worker-start-receipt import { prepareLocalWorkerStart } from './orchestration-worker-start-validation' import { resolveDispatchCreator } from './orchestration-dispatch-creator' import { resolveOrchestrationCaller } from './orchestration-run-scope' -import { - isWorkerStartTimeoutWithinTimerLimit, - resolveWorkerStartReadinessTimeoutMs -} from '../../../../shared/orchestration-timing-budgets' +import { isCallerCurrentRunCoordinator } from './orchestration-coordinator-caller' export const ORCHESTRATION_WORKER_START_METHODS: RpcMethod[] = [ defineMethod({ @@ -50,7 +51,11 @@ export const ORCHESTRATION_WORKER_START_METHODS: RpcMethod[] = [ callerEvidence: orchestrationCompatibilityEvidence }) const run = coordinatorPane ? db.getCurrentRunForPane(coordinatorPane) : undefined - if (!run || (params.run && params.run !== run.id)) { + if ( + !run || + !isCallerCurrentRunCoordinator(runtime, run, params.from, coordinatorPane) || + (params.run && params.run !== run.id) + ) { throw new OrchestrationError( 'consumer_fenced', 'worker-start requires the coordinator terminal currently bound to the Task Run.' diff --git a/src/main/runtime/rpc/methods/orchestration.ts b/src/main/runtime/rpc/methods/orchestration.ts index 4045ce11409a..83e537474fe8 100644 --- a/src/main/runtime/rpc/methods/orchestration.ts +++ b/src/main/runtime/rpc/methods/orchestration.ts @@ -29,7 +29,7 @@ import { type SendRecipientWarning } from './orchestration-recipient-routing' import { buildInjectRejectionMessage } from './orchestration-inject-rejection-message' -import { resolveRunScope } from './orchestration-run-scope' +import { assertCallerHandleMatchesEvidence, resolveRunScope } from './orchestration-run-scope' import { ORCHESTRATION_RUN_METHODS } from './orchestration-runs' import { ORCHESTRATION_WORKER_METHODS } from './orchestration-worker-methods' import { ORCHESTRATION_FEDERATION_METHODS } from './orchestration-federation-methods' @@ -38,6 +38,7 @@ import type { OrcaRuntimeService } from '../../orca-runtime' import type { RunRow } from '../../orchestration/types' import { encodeFederatedControlMessage } from '../../orchestration/federation-control-message' import { bindCoordinatorMutationPayload } from '../../orchestration/dispatch-message-binding' +import { resolveAttestedRunCoordinatorPane } from './orchestration-coordinator-caller' import { ORCHESTRATION_FEDERATION_CONTROL_MAIL_PROTOCOL_VERSION, ORCHESTRATION_FEDERATION_LIFECYCLE_SETTLEMENT_PROTOCOL_VERSION @@ -449,8 +450,10 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ runtime, orchestrationCapability, legacyCoordinatorRunId, + legacyCoordinatorAuthority, revalidateLegacyCoordinator, orchestrationCompatibilityCallerAuthority, + orchestrationCompatibilityEvidence, recordMutationReceipt, signal } @@ -561,6 +564,22 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ ...(lifecycle ? { lifecycle } : {}) } } + const lifecyclePayload = isDispatchMutationMessageType(params.type) + ? parseRemoteWorkerPayload(params.payload) + : undefined + const lifecycleDispatch = + typeof lifecyclePayload?.dispatchId === 'string' + ? db.getDispatchContextById(lifecyclePayload.dispatchId) + : undefined + const dispatchCapabilityAuthenticatesLifecycle = Boolean( + orchestrationCapability && lifecycleDispatch?.capability_hash + ) + if (!dispatchCapabilityAuthenticatesLifecycle) { + assertCallerHandleMatchesEvidence(runtime, from, orchestrationCompatibilityEvidence, { + callerAuthority: attestedCaller, + allowLegacyAuthority: Boolean(legacyCoordinatorAuthority) + }) + } const routing = resolveMessageRun(runtime, { from, senderPaneKey, @@ -1376,13 +1395,24 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ params: ReplyParams, handler: async ( params, - { orchestrationCompatibilityEvidence, runtime, legacyCoordinatorRunId } + { + orchestrationCompatibilityCallerAuthority, + orchestrationCompatibilityEvidence, + legacyCoordinatorAuthority, + legacyCoordinatorRunId, + runtime + } ) => { const db = runtime.getOrchestrationDb() const original = db.getMessageById(params.id) if (!original) { throw new Error(`Message not found: ${params.id}`) } + const from = params.from ?? original.to_handle + assertCallerHandleMatchesEvidence(runtime, from, orchestrationCompatibilityEvidence, { + callerAuthority: orchestrationCompatibilityCallerAuthority, + allowLegacyAuthority: Boolean(legacyCoordinatorAuthority) + }) if ( legacyCoordinatorRunId && (original.run_id !== legacyCoordinatorRunId || @@ -1444,10 +1474,18 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ } } + resolveRunScope(runtime, { + runId: original.run_id, + callerTerminalHandle: from, + requireCurrentConsumer: true, + legacyCoordinatorRunId, + callerEvidence: orchestrationCompatibilityEvidence + }) + db.markAsRead([original.id]) const reply = db.insertMessage({ - from: params.from ?? original.to_handle, + from, to: original.from_handle, subject: `Re: ${original.subject}`, body: params.body, @@ -1549,9 +1587,25 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ } return base }) + const callerPaneKey = params.callerTerminalHandle + ? resolveAttestedRunCoordinatorPane( + runtime, + run, + params.callerTerminalHandle, + orchestrationCompatibilityEvidence + ) + : null return { runId: run.id, legacyReadOnly: run.legacy === 1, + binding: { + currentConsumer: Boolean( + params.callerTerminalHandle && + run.legacy === 0 && + callerPaneKey !== null && + db.getCurrentRunForPane(callerPaneKey)?.id === run.id + ) + }, tasks: params.brief ? abbreviateOrchestrationTasks(tasks) : tasks, count: tasks.length } diff --git a/src/main/runtime/rpc/orchestration-11745-regression-verification.test.ts b/src/main/runtime/rpc/orchestration-11745-regression-verification.test.ts index 47d585ca244d..22a2e7ced4e9 100644 --- a/src/main/runtime/rpc/orchestration-11745-regression-verification.test.ts +++ b/src/main/runtime/rpc/orchestration-11745-regression-verification.test.ts @@ -137,7 +137,10 @@ describe('#11745 H1 — gate methods authorize the caller Run', () => { ) ) - expect(response).toMatchObject({ ok: false, error: { code: 'task_not_found' } }) + expect(response).toMatchObject({ + ok: false, + error: { code: 'consumer_fenced', data: { effectsApplied: false } } + }) expect(adoptedGraph(harness)).toEqual(before) }) @@ -175,7 +178,10 @@ describe('#11745 H1 — gate methods authorize the caller Run', () => { ) ) - expect(response).toMatchObject({ ok: false, error: { message: `Gate not found: ${gate.id}` } }) + expect(response).toMatchObject({ + ok: false, + error: { code: 'consumer_fenced', data: { effectsApplied: false } } + }) expect(harness.db.getGate(gate.id)).toMatchObject({ status: 'pending', resolution: null }) expect(adoptedGraph(harness)).toEqual(before) expect(before.task).toBe('blocked') @@ -558,7 +564,7 @@ describe('#11745 H3 — adopted Run claimed by the legacy coordinator', () => { }) describe('#11745 H4 — unattested caller on a taken-over adopted Run', () => { - it('DOCUMENTS the accepted behaviour: the binding alone grants authority', async () => { + it('requires attestation after a current coordinator takes over an adopted Run', async () => { const harness = createHarness() await legacyClaimsAdoptedRun(harness, 'v-h4') await currentCoordinatorTakesOver(harness, 'v-h4') @@ -577,8 +583,8 @@ describe('#11745 H4 — unattested caller on a taken-over adopted Run', () => { ) expect(response).toMatchObject({ - ok: true, - result: { task: { run_id: harness.adoptedRunId } } + ok: false, + error: { code: 'consumer_fenced', data: { effectsApplied: false } } }) }) diff --git a/src/main/runtime/rpc/orchestration-legacy-compatibility-dispatcher.test.ts b/src/main/runtime/rpc/orchestration-legacy-compatibility-dispatcher.test.ts index ca19570cbf56..7a9a0e225cbf 100644 --- a/src/main/runtime/rpc/orchestration-legacy-compatibility-dispatcher.test.ts +++ b/src/main/runtime/rpc/orchestration-legacy-compatibility-dispatcher.test.ts @@ -659,11 +659,71 @@ describe('legacy compatibility through RpcDispatcher', () => { ok: true, result: { runId: harness.adoptedRunId, + binding: { currentConsumer: false }, tasks: [expect.objectContaining({ id: harness.taskId })] } }) expect(counts(harness.db)).toEqual(before) - expect(harness.verify).not.toHaveBeenCalled() + expect(harness.verify).toHaveBeenCalledWith({ + terminalHandle: 'term_unrelated', + paneKey: 'tab_unrelated:55555555-5555-4555-8555-555555555555', + launchToken: 'unrelated-token' + }) + }) + + it('rejects an attested legacy coordinator that declares another sender handle', async () => { + const harness = createHarness() + const before = counts(harness.db) + const response = await harness.dispatcher.dispatch( + request( + 'orchestration.send', + { + from: 'term_spoofed', + to: WORKER_HANDLE, + run: harness.adoptedRunId, + subject: 'forged legacy coordinator mail' + }, + evidence('coordinator'), + 'legacy-coordinator-spoofed-from' + ) + ) + + expect(response).toMatchObject({ + ok: false, + error: { code: 'consumer_fenced', data: { effectsApplied: false } } + }) + expect(counts(harness.db)).toEqual(before) + expect(harness.notify).not.toHaveBeenCalled() + }) + + it('rejects a legacy direct reply that declares another sender handle', async () => { + const harness = createHarness() + const question = harness.db.createQuestion({ + runId: harness.adoptedRunId, + dispatchId: harness.dispatchId, + askerHandle: WORKER_HANDLE, + question: 'Legacy question' + }) + harness.db.db + .prepare("UPDATE messages SET delivery_contract = 'legacy_direct' WHERE id = ?") + .run(question.message.id) + const before = counts(harness.db) + + const response = await harness.dispatcher.dispatch( + request( + 'orchestration.reply', + { id: question.message.id, body: 'forged answer', from: 'term_spoofed' }, + evidence('coordinator'), + 'legacy-reply-spoofed-from' + ) + ) + + expect(response).toMatchObject({ + ok: false, + error: { code: 'consumer_fenced', data: { effectsApplied: false } } + }) + expect(counts(harness.db)).toEqual(before) + expect(harness.notify).not.toHaveBeenCalled() }) it.each(['dispatch', 'websocket'] as const)( diff --git a/src/main/runtime/rpc/orchestration-legacy-fence-jurisdiction.test.ts b/src/main/runtime/rpc/orchestration-legacy-fence-jurisdiction.test.ts index d04c24dbbd1f..30af6785f9de 100644 --- a/src/main/runtime/rpc/orchestration-legacy-fence-jurisdiction.test.ts +++ b/src/main/runtime/rpc/orchestration-legacy-fence-jurisdiction.test.ts @@ -162,7 +162,7 @@ describe('legacy coordinator fence jurisdiction', () => { expect(harness.db.listTasks({ runId: harness.adoptedRunId })).toHaveLength(1) }) - it('lets the post-takeover owner operate the adopted Run without attestation', async () => { + it('requires attestation from the post-takeover owner', async () => { const harness = createHarness() await claimAdoptedRunAsLegacyCoordinator(harness) await takeoverAdoptedRun(harness) @@ -181,8 +181,8 @@ describe('legacy coordinator fence jurisdiction', () => { ) expect(response).toMatchObject({ - ok: true, - result: { task: { run_id: harness.adoptedRunId, spec: 'owner assignment' } } + ok: false, + error: { code: 'consumer_fenced', data: { effectsApplied: false } } }) }) }) diff --git a/src/main/runtime/rpc/orchestration-legacy-mail.ts b/src/main/runtime/rpc/orchestration-legacy-mail.ts index 0af41aebd88b..bb59d085ec24 100644 --- a/src/main/runtime/rpc/orchestration-legacy-mail.ts +++ b/src/main/runtime/rpc/orchestration-legacy-mail.ts @@ -145,10 +145,24 @@ export async function handleLegacyReply(args: { if (!original || original.delivery_contract !== 'legacy_direct') { return undefined } + if (params.from && request.orchestrationCompatibilityEvidence?.terminalHandle !== params.from) { + throw new OrchestrationError( + 'consumer_fenced', + `This terminal is attested as ${request.orchestrationCompatibilityEvidence?.terminalHandle ?? 'unknown'} and cannot act as ${params.from}.`, + { effectsApplied: false } + ) + } const principal = authority.attestCoordinator(request, original.run_id) if (!principal) { throw legacyCoordinatorReadOnly() } + if (params.from && params.from !== principal.terminal_handle) { + throw new OrchestrationError( + 'consumer_fenced', + `This terminal is attested as ${principal.terminal_handle} and cannot act as ${params.from}.`, + { effectsApplied: false } + ) + } const operation = operationIdentity(request, 'reply', { questionId: params.id, body: params.body diff --git a/src/main/runtime/rpc/orchestration-legacy-question-takeover.test.ts b/src/main/runtime/rpc/orchestration-legacy-question-takeover.test.ts index 77d7e2d5a029..3d3ea154e7f2 100644 --- a/src/main/runtime/rpc/orchestration-legacy-question-takeover.test.ts +++ b/src/main/runtime/rpc/orchestration-legacy-question-takeover.test.ts @@ -279,7 +279,12 @@ describe('legacy question takeover compatibility', () => { from: CURRENT_COORDINATOR_HANDLE, run: harness.adoptedRunId }, - 'answer-after-takeover' + 'answer-after-takeover', + { + terminalHandle: CURRENT_COORDINATOR_HANDLE, + paneKey: CURRENT_COORDINATOR_PANE, + launchToken: 'current-coordinator-token' + } ) ) ).resolves.toMatchObject({ ok: true }) diff --git a/src/main/runtime/rpc/orchestration-legacy-run-routing.test.ts b/src/main/runtime/rpc/orchestration-legacy-run-routing.test.ts index 49e2b78d6aea..87b6cb3547cf 100644 --- a/src/main/runtime/rpc/orchestration-legacy-run-routing.test.ts +++ b/src/main/runtime/rpc/orchestration-legacy-run-routing.test.ts @@ -43,7 +43,7 @@ describe('legacy coordinator gate run routing', () => { } ) - it('routes an unnamed Run to the caller binding even when attestation fails', async () => { + it('does not route an unnamed Run from an unattested caller', async () => { const harness = createHarness() const run = harness.db.createRun({ objective: 'current work', @@ -64,9 +64,10 @@ describe('legacy coordinator gate run routing', () => { ) expect(response).toMatchObject({ - ok: true, - result: { task: { run_id: run.id, spec: 'fresh assignment' } } + ok: false, + error: { code: 'consumer_fenced', data: { effectsApplied: false } } }) + expect(harness.db.listTasks({ runId: run.id })).toHaveLength(0) }) it('lets a current coordinator rebind the unclaimed adopted Run with actionable guidance', async () => { diff --git a/src/main/runtime/rpc/orchestration-legacy-takeover-delivery.test.ts b/src/main/runtime/rpc/orchestration-legacy-takeover-delivery.test.ts index bcaf5fca11f6..286e36dabc0c 100644 --- a/src/main/runtime/rpc/orchestration-legacy-takeover-delivery.test.ts +++ b/src/main/runtime/rpc/orchestration-legacy-takeover-delivery.test.ts @@ -217,10 +217,10 @@ describe('legacy coordinator delivery targets after takeover', () => { ) ) - // It still owns the Run binding, so it reads its own Run rather than being fenced read-only. + // Binding metadata alone is not caller identity after the coordinator pane changes. expect(response).toMatchObject({ - ok: true, - result: { runId: harness.adoptedRunId, legacyReadOnly: false } + ok: false, + error: { code: 'consumer_fenced', data: { effectsApplied: false } } }) }) @@ -303,7 +303,7 @@ describe('legacy coordinator delivery targets after takeover', () => { }) // Why: bindRun only revokes a committed principal when it takes over or the legacy work is settled, -// so a coordinator restarting inside the legacy pane rebinds the Run while the principal stays +// so a coordinator reminting its handle inside the legacy pane rebinds while the principal stays // committed. Delivery still routes legacy_direct to the addressed handle there, and no reader can // see that mailbox, so the permit must refuse rather than accept mail nobody will ever read. describe('legacy coordinator delivery targets without a takeover', () => { @@ -329,7 +329,8 @@ describe('legacy coordinator delivery targets without a takeover', () => { harness.db.bindRun({ runId: harness.adoptedRunId, coordinatorHandle: REBOUND_COORDINATOR_HANDLE, - coordinatorPaneKey: COORDINATOR_PANE + coordinatorPaneKey: COORDINATOR_PANE, + authorityContinuity: true }) ).toMatchObject({ coordinator_handle: REBOUND_COORDINATOR_HANDLE }) expect(harness.db.getLegacyCoordinatorPrincipal(harness.adoptedRunId)?.status).toBe('committed') diff --git a/src/main/runtime/rpc/orchestration-runtime-update-settlement.test.ts b/src/main/runtime/rpc/orchestration-runtime-update-settlement.test.ts index 52205d00791e..d67f65a28c39 100644 --- a/src/main/runtime/rpc/orchestration-runtime-update-settlement.test.ts +++ b/src/main/runtime/rpc/orchestration-runtime-update-settlement.test.ts @@ -140,9 +140,7 @@ function createUpdateHarness(): Harness { runtime.setOrchestrationDb(db) vi.spyOn(runtime, 'getTerminalPaneKey').mockReturnValue(null) const getProcessIncarnation = runtime.getTerminalProcessIncarnation.bind(runtime) - vi.spyOn(runtime, 'getTerminalProcessIncarnation') - .mockImplementationOnce(getProcessIncarnation) - .mockReturnValue(null) + vi.spyOn(runtime, 'getTerminalProcessIncarnation').mockImplementation(getProcessIncarnation) vi.spyOn(runtime, 'notifyMessageArrived').mockImplementation(() => {}) return new RpcDispatcher({ runtime, methods: ORCHESTRATION_METHODS }) } diff --git a/src/main/runtime/rpc/orchestration-task-dispatch-invariant.test.ts b/src/main/runtime/rpc/orchestration-task-dispatch-invariant.test.ts index 0d194a372bae..85f7092299c8 100644 --- a/src/main/runtime/rpc/orchestration-task-dispatch-invariant.test.ts +++ b/src/main/runtime/rpc/orchestration-task-dispatch-invariant.test.ts @@ -242,6 +242,18 @@ function createHarness(): Harness { } return null }) + vi.spyOn(runtime, 'verifyOrchestrationCompatibilityCaller').mockImplementation((evidence) => + evidence?.terminalHandle === COORDINATOR_HANDLE && evidence.paneKey === COORDINATOR_PANE + ? { + terminalHandle: COORDINATOR_HANDLE, + paneKey: COORDINATOR_PANE, + processIncarnation: 'runtime:coordinator:1', + hostScope: { kind: 'local', hostId: 'local' }, + launchTokenHash: 'test-token-hash', + terminalProvenance: 'current_runtime' + } + : null + ) const runId = db.createRun({ objective: 'Enforce Task/Dispatch state', coordinatorHandle: COORDINATOR_HANDLE, @@ -369,7 +381,12 @@ function request(method: string, params: Record): RpcRequest { method, params, orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION, - orchestrationRequestId: `task_dispatch_invariant_${requestSequence}` + orchestrationRequestId: `task_dispatch_invariant_${requestSequence}`, + orchestrationCompatibilityEvidence: { + terminalHandle: COORDINATOR_HANDLE, + paneKey: COORDINATOR_PANE, + launchToken: 'test-launch-token' + } } }