From 854b2e3a35e36c2f41a82704485934a96cce644c Mon Sep 17 00:00:00 2001 From: reacher-z Date: Sun, 26 Jul 2026 07:51:42 -0700 Subject: [PATCH 1/2] feat: add cross-language durable recovery --- .github/workflows/ci.yml | 2 +- CHANGELOG.md | 18 +- CONTRIBUTING.md | 2 +- README.md | 17 +- ROADMAP.md | 18 +- codex_logs/daily/2026-07-26.md | 89 + codex_logs/task-registry.json | 98 +- docs/CONCEPTS.md | 70 +- docs/FAILURE_MODES.md | 97 +- docs/QUICKSTART.md | 50 +- docs/SECURITY.md | 64 +- package.json | 2 +- packages/cli/assets/spec/graph.schema.json | 18 +- packages/core/src/schema-validation.ts | 30 +- packages/core/test/compiler.test.ts | 20 + .../schemas/v1alpha1/graph.schema.json | 18 +- packages/persistence/src/events.ts | 1 + packages/runtime/README.md | 71 +- packages/runtime/package.json | 3 +- packages/runtime/src/durable-json.ts | 155 + packages/runtime/src/durable-types.ts | 85 + packages/runtime/src/durable.ts | 2139 ++++++++++++ packages/runtime/src/index.ts | 22 + packages/runtime/src/scheduler.ts | 507 ++- packages/runtime/src/types.ts | 3 +- packages/runtime/test/durable-json.test.ts | 113 + packages/runtime/test/durable.test.ts | 1342 ++++++++ packages/runtime/test/scheduler.test.ts | 310 ++ packages/runtime/vitest.config.ts | 3 + pnpm-lock.yaml | 3 + python/README.md | 61 +- python/src/graph_engineering/__init__.py | 20 + python/src/graph_engineering/compiler.py | 20 +- python/src/graph_engineering/durable.py | 2130 ++++++++++++ python/src/graph_engineering/durable_json.py | 126 + python/src/graph_engineering/events.py | 8 +- python/src/graph_engineering/models.py | 50 +- python/src/graph_engineering/scheduler.py | 495 ++- python/tests/test_compiler.py | 83 + python/tests/test_durable_json.py | 102 + python/tests/test_durable_scheduler.py | 2856 +++++++++++++++++ python/tests/test_events.py | 11 + python/tests/test_scheduler.py | 71 + scripts/validate-fixtures.mjs | 56 +- spec/README.md | 4 + spec/conformance/durable-json.case.json | 233 ++ spec/conformance/durable-resume.case.json | 94 + spec/conformance/expected.json | 8 + .../invalid-oversized-timers.graph.json | 26 + .../invalid-unsafe-budget.graph.json | 20 + spec/conformance/strict-rfc3339.case.json | 30 + spec/durable-json.schema.json | 103 + spec/durable-recovery-semantics.md | 376 +++ spec/event.schema.json | 3 +- spec/graph.schema.json | 18 +- spec/persistence-semantics.md | 6 +- spec/runtime-semantics.md | 2 + tools/conformance/python_durable_interop.py | 145 + tools/conformance/python_durable_report.py | 157 + .../conformance/python_persistence_report.py | 9 +- tools/conformance/python_primitives_report.py | 1 - tools/conformance/python_report.py | 12 +- tools/conformance/python_router_report.py | 1 - .../python_runtime_extended_report.py | 9 +- tools/conformance/python_runtime_report.py | 10 +- tools/conformance/run.mjs | 296 +- 66 files changed, 12662 insertions(+), 360 deletions(-) create mode 100644 packages/runtime/src/durable-json.ts create mode 100644 packages/runtime/src/durable-types.ts create mode 100644 packages/runtime/src/durable.ts create mode 100644 packages/runtime/test/durable-json.test.ts create mode 100644 packages/runtime/test/durable.test.ts create mode 100644 python/src/graph_engineering/durable.py create mode 100644 python/src/graph_engineering/durable_json.py create mode 100644 python/tests/test_durable_json.py create mode 100644 python/tests/test_durable_scheduler.py create mode 100644 spec/conformance/durable-json.case.json create mode 100644 spec/conformance/durable-resume.case.json create mode 100644 spec/conformance/invalid-oversized-timers.graph.json create mode 100644 spec/conformance/invalid-unsafe-budget.graph.json create mode 100644 spec/conformance/strict-rfc3339.case.json create mode 100644 spec/durable-json.schema.json create mode 100644 spec/durable-recovery-semantics.md create mode 100644 tools/conformance/python_durable_interop.py create mode 100755 tools/conformance/python_durable_report.py mode change 100644 => 100755 tools/conformance/python_persistence_report.py mode change 100644 => 100755 tools/conformance/python_report.py mode change 100644 => 100755 tools/conformance/python_runtime_extended_report.py mode change 100644 => 100755 tools/conformance/python_runtime_report.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1940d2e..42402ba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -98,7 +98,7 @@ jobs: - run: uv run --project python --python ${{ matrix.python }} pytest - run: uv run --project python --python ${{ matrix.python }} python examples/quickstart/run.py - run: uv run --project python --python ${{ matrix.python }} ruff check python/src python/tests - - run: uv run --project python --python ${{ matrix.python }} mypy python/src + - run: uv run --project python --python ${{ matrix.python }} mypy --config-file python/pyproject.toml python/src - if: matrix.python == '3.13' run: uv build --project python - if: matrix.python == '3.13' diff --git a/CHANGELOG.md b/CHANGELOG.md index 08d693b..9cf763b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,9 +6,25 @@ migration note. ## [Unreleased] +### Added + +- Event-sourced durable start and resume APIs in TypeScript and Python. Runs bind + graph, original input, and caller-supplied implementation identity; node + outcomes commit before releasing dependants; committed successes are reused + after process loss. +- Cross-language tagged Durable JSON for exact finite binary64 payload hashing, + stable activity/idempotency keys, terminal-result snapshots, and recovery + conformance fixtures. +- Fail-closed interrupted-attempt handling: nodes declared with no or idempotent + side effects may retry within their original budgets, while omitted or + non-idempotent declarations surface `IN_DOUBT_SIDE_EFFECT`. Resuming a valid + terminal run returns its recorded result with no new event or executor call. + ### In progress -- Scheduler-integrated event emission, checkpoint recovery, replay, and fork. +- Scheduler checkpoint acceleration, replay/fork, and distributed lease/fencing + providers. Recovery correctness currently comes from the complete event + stream; checkpoint files are not wired into the scheduler. - Streaming pipelines, conditional edge lowering, verifier panels, and bounded runtime loops. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4c5859f..7c3eb77 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -28,7 +28,7 @@ Python uses uv: uv sync --project python --extra dev uv run --project python pytest python/tests uv run --project python ruff check python/src python/tests -uv run --project python mypy python/src +uv run --project python mypy --config-file python/pyproject.toml python/src ``` Changes affecting the Graph IR, compiler diagnostics, event protocol, or diff --git a/README.md b/README.md index 7912889..153766c 100644 --- a/README.md +++ b/README.md @@ -28,9 +28,11 @@ The project is an early alpha. The DAG compiler, ready-queue schedulers, safe pr initializer, machine-readable CLI, structured failure handling, retries, timeouts, budgets, settled barrier/router decisions, safe Mermaid/DOT rendering, pattern constructors, local event/checkpoint stores, read-only MCP server, and -development progress scanner are executable today. Scheduler-integrated durable -recovery and the broader v1 surface remain under active development; the -repository does not silently mock unfinished capabilities. +development progress scanner are executable today. Both native runtimes also +provide event-sourced durable start/resume: committed successes are reused after +process loss and unsafe ambiguous effects fail closed. Checkpoint acceleration, +replay/fork, distributed leases, and the broader v1 surface remain under active +development; the repository does not silently mock unfinished capabilities. > **Source-only alpha:** npm and PyPI packages are not published yet. Clone this > repository to try the current release candidate; registry publication remains @@ -95,7 +97,8 @@ uv run --project python pytest python/tests | Diamond/verifier pattern constructors | Yes | Consumes the portable Graph IR | | Safe Mermaid/DOT visualization | Yes, through the CLI | Same portable Graph IR | | Local event/checkpoint stores | Yes | Yes | -| Scheduler checkpoint/resume | In development | In development | +| Event-sourced scheduler start/resume | Yes | Yes | +| Scheduler checkpoint acceleration | Not yet | Not yet | | Read-only validation/planning MCP | Yes | Uses the same portable IR | | Streaming and scheduler-applied routers/verifier panels/loops | Target v1 | Target v1 | @@ -103,7 +106,8 @@ uv run --project python pytest python/tests - Explicit node and edge data contracts. - Parallel, pipeline, barrier, router, verifier, and bounded-loop topologies. -- Durable checkpoints, resume, replay, and fork. +- Durable event-sourced start/resume today; rebuildable checkpoint acceleration, + replay, and fork as explicit follow-up protocols. - Deterministic plumbing; models are reserved for judgment. - Provider-neutral adapters and deny-by-default capabilities. - Observable runs with portable events and traces. @@ -123,7 +127,7 @@ corepack pnpm check:packages corepack pnpm check:packed-install corepack pnpm audit:prod uv run --project python ruff check python/src python/tests -uv run --project python mypy python/src +uv run --project python mypy --config-file python/pyproject.toml python/src uv build --project python python3 scripts/check-python-artifacts.py ``` @@ -139,6 +143,7 @@ python3 scripts/check-python-artifacts.py - [Support](SUPPORT.md) - [Runtime semantics](spec/runtime-semantics.md) - [Persistence semantics](spec/persistence-semantics.md) +- [Durable recovery semantics](spec/durable-recovery-semantics.md) - [Primitive semantics](spec/primitives-semantics.md) - [21-day delivery plan](codex_plans/Graph-Engineering-21-Day-Master-Plan.md) diff --git a/ROADMAP.md b/ROADMAP.md index bc698e9..adbef3f 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -18,15 +18,19 @@ earned by executable cross-language tests. ## Alpha 2 — durable graph patterns -- Scheduler event emission and checkpoint-safe node commit. -- Crash recovery that reuses completed node results. -- Replay and fork with immutable lineage. -- Pipeline buffers/backpressure and explicit barrier policies. -- Deterministic routers, verifier verdicts, quorum/unknown outcomes, reflection, +- [x] Scheduler event emission with commit-before-release node outcomes and + tagged Durable JSON payloads. +- [x] Event-sourced crash continuation that reuses committed successes, preserves + attempt budgets, and fails closed for ambiguous unsafe effects. +- [ ] Scheduler checkpoint acceleration; correctness already rebuilds from the + authoritative event history. +- [ ] Replay and fork with immutable lineage. +- [ ] Pipeline buffers/backpressure and explicit barrier policies. +- [ ] Deterministic routers, verifier verdicts, quorum/unknown outcomes, reflection, and bounded loop-until-dry primitives. -- Deterministic mock, OpenAI, Anthropic, Gemini, OpenAI-compatible, HTTP, shell, +- [ ] Deterministic mock, OpenAI, Anthropic, Gemini, OpenAI-compatible, HTTP, shell, and MCP adapters behind explicit capability declarations. -- Ten executable TypeScript/Python patterns with failure and recovery fixtures. +- [ ] Ten executable TypeScript/Python patterns with failure and recovery fixtures. ## Beta — operations and external validation diff --git a/codex_logs/daily/2026-07-26.md b/codex_logs/daily/2026-07-26.md index 1c6a691..fbda595 100644 --- a/codex_logs/daily/2026-07-26.md +++ b/codex_logs/daily/2026-07-26.md @@ -147,3 +147,92 @@ Canonical IR -> language models/builders -> compiler -> deterministic scheduler code-review/security graphs, ready-queue benchmarking, and honest adopter stories. - Verified anonymously that the repository, release, discussions, issue chooser, private vulnerability report form, and raw README each return HTTP 200. + +## 12:42 UTC durable-recovery design checkpoint + +- Confirmed that both CI and CodeQL passed again for the public-release log + commit `d242b65`; the protected public alpha is green before feature work. +- Created branch `feat/durable-recovery` and ran three parallel read-only audits + over the TypeScript scheduler, Python scheduler, persistence stores, recovery + state machine, crash windows, and shared conformance requirements. +- Froze the first scheduler-integrated recovery contract around an authoritative + CAS event history, commit-before-dependent-release, immutable graph/input/ + implementation bindings, stable activity keys, terminal idempotence, and + conservative handling of unknown external effects. Checkpoints remain + rebuildable acceleration rather than a second source of truth. +- Added a shared diamond recovery case in which one committed branch is reused, + one interrupted safe branch advances to attempt two, the merge receives both + outputs, total attempt budgets survive restart, and a second terminal resume + produces zero events and zero executor calls. +- Started parallel native implementation lanes and a separate exact Tagged + Durable JSON vector/schema lane. Tagged values preserve finite binary64 data + across TypeScript/Python while staying inside checkpoint safe-integer rules. +- Registry publication remains deferred. This feature will be committed as + `reacher-z ` without co-author trailers only after focused, + cross-language, full-suite, packaging, and security validation passes. + +## 13:41 UTC durable-recovery adversarial checkpoint + +- Independently reran the Python lane after its first implementation: 419 tests, + Ruff, strict mypy, and diff hygiene passed. Core and persistence remain green + at 36 and 27 tests, and all 14 Tagged Durable JSON vectors still validate. +- Two independent read-only audits then found actionable protocol defects before + commit: zero-attempt node outcomes existed only in a terminal snapshot; + Python could trust a shallowly frozen compiled graph; normal retry admission + ignored the global attempt budget; and malformed retry timestamps could append + `RunResumed` before failing validation. +- Froze `NodeSettledWithoutAttempt` as an explicit event-derived outcome for + missing-executor, binding, upstream, budget, and pre-dispatch cancellation + paths. Both schedulers must commit it before releasing dependants, and terminal + folding must derive every node result and both orders from history. +- Hardened the TypeScript scheduler so a journal failure is not held hostage by + an executor that ignores cancellation, and public `runGraph` now runtime- + whitelists options so JavaScript callers cannot inject internal journals or + recovery seeds. A dedicated scheduler lane is adding atomic global retry + reservations and settlement hooks with concurrency tests. +- Added safe-integer limits to the shared Graph IR schema and TypeScript shape + validation, expanded the portable event vocabulary, and started a + cross-language durable report. Its Python side already proves committed-left + reuse, interrupted-right attempt advancement, stable activity keys, merge + input parity, four total attempts, and terminal zero-event/zero-executor + idempotence. +- The thirty-minute timer remains active. A manual scan after registering the + conformance lane reports 30 tasks: 29 healthy and one waiting only on the two + active native implementations; no stale, blocked, warning, or integration-risk + item was reported. + +## 14:49 UTC durable-recovery completion checkpoint + +- Completed native event-sourced `start` and `resume` APIs in TypeScript and + Python. The append-only CAS event stream is authoritative; graph, input, and + implementation identities are bound at creation; attempt claims commit before + executor work; outcomes commit before dependants are released; and a valid + terminal resume performs zero writes and zero executor calls. +- Added exact Tagged Durable JSON for finite binary64 values, stable activity + keys, strict real-calendar RFC 3339 timestamps, unique per-run event IDs, + explicit zero-attempt settlements, portable failure identities, 32-bit host + timer bounds, and fail-closed recovery for ambiguous unsafe side effects. +- Closed adversarial state-machine cases covering duplicate or stale starts, + pending retry reservations, global attempt races, premature downstream + settlements, prototype-named JSON keys, mutable executor registries and graph + views, event factory/clock failures, completion commit order, and non-cooperative + Python handlers that swallow cancellation after a durability failure. +- Expanded conformance from Python-produced interrupted history resumed by + TypeScript to bidirectional terminal-history consumption for missing executors + and output-binding failures. Each cross-language terminal replay reconstructs + the recorded result with zero new events and zero executor calls. +- Final local release gates are green: Core 37 tests, Runtime 81 tests, Python + 486 tests, all workspace tests, build/typecheck/lint, strict mypy, both Ruff + configurations, eight compiler fixtures, runtime/primitives/persistence and + durable cross-language conformance, seven npm tarballs and clean installs, + Python wheel/sdist audits, source and isolated quickstarts, documentation links, + fixture validation, production dependency audit, and diff hygiene. +- Synchronized the canonical Graph Schema into CLI and MCP packaged assets and + corrected CI/documented mypy commands so they explicitly load the Python + strict configuration. The 07:40 automatic progress scan reported every + completed lane healthy and the remaining conformance lane recently active; + the post-gate manual scan then reported 30 of 30 tasks healthy with zero + waiting, stale, blocked, warning, or integration-risk items. +- Distributed leases/fencing, checkpoint acceleration, replay/fork, approval + callbacks, and universal exactly-once effects remain deliberately unclaimed + follow-up work. Registry publication remains deferred. diff --git a/codex_logs/task-registry.json b/codex_logs/task-registry.json index c3ae6c0..2e857d9 100644 --- a/codex_logs/task-registry.json +++ b/codex_logs/task-registry.json @@ -1,6 +1,6 @@ { "schema_version": 1, - "updated_at": "2026-07-26T12:20:00Z", + "updated_at": "2026-07-26T13:40:44Z", "repository": ".", "scan_policy": { "warning_after_minutes": 60, @@ -428,6 +428,102 @@ "risk": "medium", "blocker": null, "next_action": "Use the 9,416+ moving parity benchmark, finish GitHub launch settings, and execute the adoption/content backlog without promising stars" + }, + { + "id": "D6-DURABLE-SPEC-010", + "title": "Cross-language durable scheduler recovery contract and fixtures", + "owner": "main + /root/durable_contract", + "status": "completed", + "work_package": "WP3", + "depends_on": ["D1-SPEC-001", "D2-TS-PERSIST-001", "D2-PY-PERSIST-001"], + "expected_artifacts": [ + "spec/durable-recovery-semantics.md", + "spec/durable-json.schema.json", + "spec/conformance/durable-json.case.json", + "spec/conformance/durable-resume.case.json" + ], + "expected_tests": [ + "tagged Durable JSON hash parity", + "committed-success reuse", + "interrupted-attempt recovery", + "terminal resume idempotence" + ], + "assigned_at": "2026-07-26T12:23:00Z", + "started_at": "2026-07-26T12:23:00Z", + "last_heartbeat": "2026-07-26T13:40:44Z", + "risk": "high", + "blocker": null, + "next_action": "Hold the contract stable while both native implementations and the shared recovery harness are verified" + }, + { + "id": "D6-TS-DURABLE-010", + "title": "TypeScript event-sourced scheduler start and resume", + "owner": "/root/durable_contract (handoff from /root/durable_ts)", + "status": "completed", + "work_package": "WP3", + "depends_on": ["D6-DURABLE-SPEC-010", "D3-TS-RUNTIME-002", "D2-TS-PERSIST-001"], + "expected_artifacts": ["packages/runtime"], + "expected_tests": [ + "commit-before-release", + "successful node never reruns", + "attempt budgets survive crash", + "identity/history/CAS rejection", + "terminal idempotence" + ], + "assigned_at": "2026-07-26T12:42:00Z", + "started_at": "2026-07-26T12:42:00Z", + "last_heartbeat": "2026-07-26T14:39:43Z", + "risk": "high", + "blocker": null, + "next_action": "Maintain the 81-test durable runtime contract while shared release and package gates run" + }, + { + "id": "D6-PY-DURABLE-010", + "title": "Python event-sourced scheduler start and resume", + "owner": "/root/durable_py", + "status": "completed", + "work_package": "WP3", + "depends_on": ["D6-DURABLE-SPEC-010", "D3-PY-RUNTIME-002", "D2-PY-PERSIST-001"], + "expected_artifacts": ["python/src/graph_engineering"], + "expected_tests": [ + "commit-before-release", + "successful node never reruns", + "attempt budgets survive crash", + "identity/history/CAS rejection", + "terminal idempotence" + ], + "assigned_at": "2026-07-26T12:42:00Z", + "started_at": "2026-07-26T12:42:00Z", + "last_heartbeat": "2026-07-26T14:39:43Z", + "risk": "high", + "blocker": null, + "next_action": "Maintain the 486-test durable runtime contract as the branch enters CI review" + }, + { + "id": "D6-DURABLE-CONFORMANCE-011", + "title": "Cross-language crash/resume and adversarial history conformance", + "owner": "main", + "status": "completed", + "work_package": "WP3-WP11", + "depends_on": ["D6-TS-DURABLE-010", "D6-PY-DURABLE-010"], + "expected_artifacts": [ + "tools/conformance/python_durable_report.py", + "tools/conformance/python_durable_interop.py", + "tools/conformance/run.mjs" + ], + "expected_tests": [ + "Python-created crash history resumes in TypeScript", + "shared result and event sequence parity", + "terminal zero-event zero-executor resume", + "resigned history rejection", + "full repository and package gates" + ], + "assigned_at": "2026-07-26T13:28:00Z", + "started_at": "2026-07-26T13:28:00Z", + "last_heartbeat": "2026-07-26T14:49:21Z", + "risk": "high", + "blocker": null, + "next_action": "Monitor protected-branch CI and review feedback without weakening the recovery contract" } ] } diff --git a/docs/CONCEPTS.md b/docs/CONCEPTS.md index 209984e..f6776c2 100644 --- a/docs/CONCEPTS.md +++ b/docs/CONCEPTS.md @@ -14,10 +14,10 @@ behavior that exists today and the intended v1 architecture. | Area | Current alpha | Target v1 | | --- | --- | --- | | Portable format | Versioned JSON Graph IR, canonical hash, compiler diagnostics, conformance fixtures | Stable compatibility and migration policy | -| Native runtimes | TypeScript and Python IR, compilers, and in-memory ready-queue schedulers pass the shared diamond and no-layer-barrier runtime conformance cases; APIs remain unstable | Durable/distributed execution and a broader cross-language conformance corpus | +| Native runtimes | TypeScript and Python IR, compilers, ready-queue schedulers, and event-sourced start/resume pass shared conformance cases; APIs remain unstable | Distributed execution and a broader cross-language conformance corpus | | Topologies | DAG fan-out/fan-in; deterministic settled all/minimum/percentage evaluation; TypeScript constructors for diamonds, verifier fan-out, declarative routing, and finite loop expansion | Streaming pipelines, scheduler-applied routing, verifier policies, quorum/deadline barriers, subgraphs, and dynamic bounded loops | -| Persistence | Native local JSONL event and atomic checkpoint stores implement the shared CAS/hash contract; schedulers remain in-memory and are not wired to recovery | Scheduler-integrated resume, replay, fork, leases, artifact stores, and production databases | -| Security | Graph bounds and explicit side-effect metadata; executors still have the ambient authority of their process | Enforced capabilities, worktree/process/container isolation, redaction, approval gates, and policy-audited dynamic graphs | +| Persistence | Native local event stores drive scheduler start/resume from authoritative history; atomic checkpoint stores exist separately but do not accelerate the scheduler | Checkpoint acceleration, replay, fork, leases, artifact stores, and production databases | +| Security | Graph bounds and a trusted `sideEffects` declaration gate ambiguous durable retries; executors still have ambient process authority | Enforced capabilities, worktree/process/container isolation, redaction, approval gates, and policy-audited dynamic graphs | “Target v1” is a design commitment, not a claim that the feature is already available. See the [runtime package status](../packages/runtime/README.md) for the @@ -256,48 +256,56 @@ critical-path cost estimation remain target-v1 capabilities. ## Durable execution: the semantic boundary -The current TypeScript and Python schedulers are in-memory. A process crash loses -their run progress, and rerunning starts a new execution. Separate native local -persistence packages now provide last-sequence-CAS event streams and atomic, -content-hashed checkpoints, but the schedulers do not yet emit those events or -resume from those checkpoints. Storage primitives are not recovery by -themselves. +Both native runtimes now expose separate event-sourced start and resume +operations in addition to their existing in-memory scheduler APIs. Start creates +a new run and never silently resumes one. Resume requires an existing run, reads +its original input from `RunCreated`, and never silently creates or replaces it. +Each run binds the compiler graph hash, input hash, effective attempt budget, and +a hash of the caller-supplied implementation identity. -The target-v1 durable contract is stricter than periodically serializing memory: +The v1alpha1 durable contract is stricter than periodically serializing memory: - an append-only run event log is the source of truth; - every event has a monotonically increasing sequence per run; - appends use an expected prior version to detect concurrent writers; - a validated node result is durable before a dependant becomes schedulable; -- checkpoints accelerate reconstruction but do not replace event history; - resume reuses recorded successful activity results; -- replay uses recorded nondeterministic results instead of calling the outside - world again; -- fork creates a new linked history rather than rewriting the old one. - -The first three storage-level rules are executable in both languages and pass a -shared checkpoint hash vector. The remaining scheduler-level rules define what -recovery must mean before resume can be claimed. See the -[persistence semantics](../spec/persistence-semantics.md); the existence of a -store is not evidence that the alpha scheduler already persists progress. +- an attempt claim is committed before executor code runs, and a success plus + its ordered edge emissions is committed before dependants are released; +- a valid terminal resume returns the recorded result with no new event, + checkpoint write, or executor call. + +Recovery correctness currently rebuilds from the complete event stream. +Content-hashed checkpoint adapters exist, but scheduler checkpoint acceleration +is not implemented. Replay, fork, dynamic graph changes, and distributed +lease/fencing are also outside this slice. CAS rejects stale event appends but is +not a distributed lease, so an application must stop the old coordinator before +resuming a run. See the +[durable recovery semantics](../spec/durable-recovery-semantics.md) and +[persistence semantics](../spec/persistence-semantics.md). ### Exactly-once stops at the external boundary -The system can make an internal checkpointed result effectively once. It cannot -atomically commit both its store and an arbitrary email, payment, shell command, -or third-party API. A crash can occur after the external effect succeeds but -before local success is recorded. - -Therefore target-v1 external activities are **at least once**: +The scheduler can make a committed internal result authoritative for a run. It +cannot atomically commit both its event store and an arbitrary email, payment, +shell command, or third-party API. A crash can occur after the external effect +succeeds but before local success is recorded. External activities are therefore +**at least once**: - idempotent activities reuse a stable logical-activity key across retries; -- an activity ledger records request and result evidence; -- non-idempotent recovery requires explicit policy or human confirmation; - documentation must never promise universal exactly-once effects. -Node `sideEffects` metadata exists in Graph IR, but the alpha executor does -not yet enforce idempotency or approval. Treat every custom executor according to -the actual external system it calls. +For an open attempt after process loss, durable resume automatically retries only +nodes declared `sideEffects: "none"` or `sideEffects: "idempotent"`, within the +original per-node and global attempt budgets. An idempotent executor receives the +same activity key, but the application must actually forward it to the external +system. An omitted or `"non-idempotent"` declaration fails closed with +`IN_DOUBT_SIDE_EFFECT` and is not invoked again. + +The runtime cannot verify that a claimed idempotent operation really is +idempotent. It has no durable activity ledger, reconciliation engine, or approval +callback. Those controls, along with an auditable non-idempotent recovery +protocol, remain application responsibilities and target-v1 work. ## State, artifacts, and isolation diff --git a/docs/FAILURE_MODES.md b/docs/FAILURE_MODES.md index ae1eb0b..18428b2 100644 --- a/docs/FAILURE_MODES.md +++ b/docs/FAILURE_MODES.md @@ -7,20 +7,24 @@ must handle. ## Status and guarantees -The current TypeScript and Python runtimes are **in-memory DAG schedulers**. Both -native implementations pass the shared diamond and no-layer-barrier ready-queue -conformance cases, bound concurrency/attempts, retry and time out node attempts, -preserve structured failures, skip affected descendants, and let independent -branches continue. Both expose cooperative run cancellation and reject graph -inputs or node results that are not detached portable finite JSON. - -Native local event and checkpoint stores now exist, but the schedulers do not yet -write or recover from them. They do not yet provide scheduler-integrated resume, -streaming pipelines, conditional routing, verifier panels, explicit loop -primitives, dynamic graph patches, distributed leases, provider rate limiting, -worktree isolation, or capability enforcement. Sections about those features are -marked **target v1** and are operational requirements, not claims about current -code. +The TypeScript and Python runtimes provide both an ordinary in-memory DAG +scheduler and separate event-sourced start/resume operations. Both native +implementations pass shared ready-queue and durable-recovery conformance cases, +bound concurrency/attempts, retry and time out node attempts, preserve structured +failures, skip affected descendants, and let independent branches continue. Both +expose cooperative run cancellation and reject graph inputs or node results that +are not detached portable finite JSON. + +Durable runs write authoritative scheduler events and reconstruct continuation +from the complete event history. They bind graph, original input, and +caller-supplied implementation identity, reuse committed successes, preserve +consumed attempt budgets, and make terminal resume side-effect free. Standalone +checkpoint stores exist, but scheduler checkpoint acceleration does not. Replay, +fork, dynamic graph patches, distributed leases, streaming pipelines, +conditional routing, verifier panels, explicit loop primitives, provider rate +limiting, worktree isolation, and capability enforcement also remain future +work. Sections marked **target v1** are operational requirements, not current +claims. ## Failure is data @@ -187,8 +191,12 @@ idempotency key for the logical activity and reuse it across attempts. Query the external system before retrying an ambiguous result. Mark truly non-idempotent nodes and require review or approval on recovery. -The current `sideEffects` field is metadata only; the alpha runtime does not -enforce idempotency or approvals. +The ordinary in-memory scheduler treats `sideEffects` as descriptive metadata. +Durable resume uses it only to decide whether an open, unknown-outcome attempt +may be invoked again: `none` and `idempotent` are eligible within the original +budgets, while an omitted or `non-idempotent` declaration fails closed with +`IN_DOUBT_SIDE_EFFECT`. This policy cannot prove that an external operation is +actually idempotent, and there is no approval callback. ### Timeout does not terminate executor code @@ -321,40 +329,57 @@ the eventual durable implementation. Track rubric score and artifact hash. Stop on threshold, no improvement, repeated state, budget, or iteration limit. Do not equate “different output” with progress. -## Durable execution failures (scheduler integration target v1) +## Durable execution failures -The current scheduler restarts from the beginning after process loss even though -standalone local event/checkpoint adapters are available. The following are -target-v1 recovery requirements. +### Crash after effect, before the durable outcome -### Crash after effect, before checkpoint - -The external effect may exist while local state says it is incomplete. Reuse a -stable activity idempotency key, record request intent before dispatch where -appropriate, reconcile ambiguous activities, and gate non-idempotent retries. +The external effect may exist after `NodeStarted` commits while the event stream +still has no attempt outcome. Resume records the interrupted attempt as consumed. +It may retry a node declared `sideEffects: "none"` or `"idempotent"`; idempotent +attempts receive the same stable activity key. For omitted or non-idempotent +declarations it throws `IN_DOUBT_SIDE_EFFECT` without invoking the executor +again. Applications must still forward the key, reconcile ambiguous remote +state, and authorize any compensation. External exactly-once execution is not +provided. ### Crash after one branch succeeds -Persist each validated node result immediately; do not wait for an entire visual -layer or barrier. Resume must reuse that result and schedule only unfinished work. +The current durable scheduler commits each validated `NodeSucceeded` together +with its ordered outgoing `EdgeEmitted` facts before releasing dependants. Resume +reuses that result and schedules only unfinished work; it does not wait for an +entire visual layer or barrier before persisting progress. ### Two orchestrators resume one run -Use a lease plus compare-and-swap event append. A stale owner cannot continue -after losing its lease. Split-brain execution is especially dangerous for side -effects; detecting it after both workers write is too late. +The current continuation claim appends `RunResumed` with compare-and-swap before +calling an executor, so competing resume calls cannot both commit that claim. +CAS is not a lease or fencing token: an old coordinator or two processes may +still execute external work before one loses a write race. Ensure the old +coordinator has stopped before resume. A lease/fencing provider that stops stale +owners is target-v1 work. ### Code or graph changes during resume -Bind each run to an immutable graph revision/hash and activity implementation -version. Resume the original revision. Replay or fork under changed code must be -an explicit operation with compatibility checks, not an invisible upgrade. +Current start binds each run to graph revision/hash, original input hash, and a +hash of the caller-supplied `implementationId`; resume rejects mismatches before +invoking executors. The implementation ID is a caller assertion, not code +attestation. Replay or fork under changed code is not implemented and must +eventually be an explicit operation rather than an invisible upgrade. ### Checkpoint is mistaken for truth -Treat the append-only event history as the source of truth and checkpoints as -reconstruction accelerators. Validate checkpoint hash/version, and rebuild from -events when it is missing or corrupt. +The append-only event history is the current source of truth and resume folds it +in full. Local checkpoint adapters are not connected to scheduler recovery, so a +checkpoint cannot authorize or change continuation. Future checkpoint +acceleration must validate its history position and projection, ignore stale or +corrupt caches, and remain rebuildable from events. + +### Terminal resume repeats completed work + +A valid terminal history is idempotent in both native runtimes: resume returns +the recorded graph result with zero new events, zero checkpoint writes, and zero +executor calls. A terminal snapshot that contradicts folded node history is +`INVALID_RUN_HISTORY`, not a reason to trust the snapshot or rerun work. ## Security and isolation failures diff --git a/docs/QUICKSTART.md b/docs/QUICKSTART.md index fe154a4..aca68cf 100644 --- a/docs/QUICKSTART.md +++ b/docs/QUICKSTART.md @@ -68,6 +68,48 @@ The Python runtime executes the identical Graph IR natively: uv run --project python python examples/quickstart/run.py ``` +## Continue a durable run after process loss + +The library APIs also support event-sourced continuation. TypeScript uses +`startDurableGraphRun` for a new stream and `resumeDurableGraphRun` for an +existing stream; Python exposes the equivalent `start_graph_run` and +`resume_graph_run`. Start and resume never silently substitute for one another. + +```ts +import { JsonlEventStore } from "@graph-engineering/persistence"; +import { + resumeDurableGraphRun, + startDurableGraphRun, +} from "@graph-engineering/runtime"; + +const eventStore = new JsonlEventStore({ directory: ".graph-engineering" }); +const options = { + runId: "research-001", + implementationId: "research-handlers@1", + eventStore, + nodeExecutors: { + scope: () => ({ topic: "graphs" }), + "research-docs": () => ({ finding: "document the contract" }), + "research-code": () => ({ finding: "test the runtime" }), + synthesize: () => ({ summary: "graph engineering" }), + }, +}; + +// Invoke with --resume only in a replacement process after confirming that the +// former coordinator stopped. +const result = process.argv.includes("--resume") + ? await resumeDurableGraphRun(graph, options) + : await startDurableGraphRun(graph, { topic: "graphs" }, options); +``` + +If the run already reached a terminal event, resume simply returns its recorded +result with no new event and no executor call. After a real interrupted attempt, +automatic retry is limited to nodes declared `sideEffects: "none"` or +`"idempotent"`; omitted and non-idempotent declarations fail closed. See the +[runtime package guide](../packages/runtime/README.md) and +[durable recovery contract](../spec/durable-recovery-semantics.md) before using +this alpha API with external effects. + ## Automation-friendly output Both commands support stable JSON: @@ -111,7 +153,7 @@ invalid graph. This CLI slice accepts canonical Graph IR as JSON and implements `init`, `validate`, `plan`, `compile`, and `doctor`. Planning and compilation are read-only: they do not call a provider or pretend that a model ran. Native -schedulers and standalone local persistence adapters are available as library -APIs, but are not exposed by this Quickstart command flow. YAML input, -scheduler-integrated recovery/replay, and the Web Explorer are subsequent public -slices. +schedulers, local persistence adapters, and event-sourced start/resume are +available as library APIs, but are not exposed by this CLI command flow. YAML +input, scheduler checkpoint acceleration, replay/fork, distributed leases, and +the Web Explorer are subsequent public slices. diff --git a/docs/SECURITY.md b/docs/SECURITY.md index d8bcfcf..05b3cdd 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -11,8 +11,9 @@ issue. ## Alpha warning This project is **not production hardened**. Current packages establish Graph IR, -structural compilation, bounded native schedulers, and structured failures. They -do not yet provide a sandbox or a complete security policy engine. +structural compilation, bounded native schedulers, structured failures, and +event-sourced continuation. They do not yet provide a sandbox or a complete +security policy engine. In particular, today: @@ -20,8 +21,9 @@ In particular, today: - a Python node handler runs inside the host Python process; - executor code can inherit the process environment, filesystem, network, and subprocess authority; -- Graph IR `resources`, `isolation`, and `sideEffects` fields are descriptive and - are not capability enforcement; +- Graph IR `resources` and `isolation` fields are descriptive; `sideEffects` is + a trusted declaration used to gate durable retry, not verified capability or + idempotency enforcement; - timeout and cancellation are cooperative controls, not containment boundaries; - worktree, process, and container isolation providers are target-v1 work; - scoped secret injection, payload redaction, human approval, authenticated @@ -113,6 +115,12 @@ Current controls are useful, but none is a substitute for process isolation: identifiers, fsync, and corruption detection; - native checkpoint stores use safe identifiers, atomic replacement, strict portable state, and verified content hashes; +- native durable schedulers bind graph/input/implementation identity, commit an + attempt claim before executor dispatch, commit successful outcomes before + releasing dependants, and rebuild continuation from scheduler events; +- durable terminal resume returns the recorded result without appending an event + or invoking an executor; ambiguous open attempts declared non-idempotent or + without a side-effect class fail closed with `IN_DOUBT_SIDE_EFFECT`; - the MCP server exposes only bounded validation, planning, and a fixed bundled schema over local stdio; it has no run, file, network, or shell tool. @@ -128,8 +136,11 @@ Important limitations: - graph hashes provide identity, not author authenticity or authorization; - an executor can bypass Graph IR metadata and directly use ambient process privileges; -- local JSONL history is durable after a successful fsync, but it is not - authenticated or tamper-evident and is not connected to scheduler recovery. +- local JSONL history is durable after a successful fsync and is the current + scheduler recovery source, but it is neither authenticated nor tamper-evident; + an attacker with write access can forge data and recompute unkeyed hashes; +- checkpoint files are not connected to scheduler recovery and provide no + authorization or resume claim. ## Capability model (target v1) @@ -300,7 +311,17 @@ apply their own allowlists. ## External side effects and recovery External mutation is at least once. A crash can occur after the remote service -commits but before the orchestrator records success. Target-v1 handling requires: +commits but before the orchestrator records success. + +Current durable recovery commits `NodeStarted` before dispatch and gives every +attempt for one logical node activity the same activity/idempotency key. After +process loss, an open attempt declared `sideEffects: "none"` or +`"idempotent"` may retry within the original budgets. An omitted or +`"non-idempotent"` declaration fails closed with `IN_DOUBT_SIDE_EFFECT`; the +runtime does not reinvoke that executor. The application must actually forward +the key to the remote API and must not mislabel an operation as idempotent. + +Safe production handling additionally requires: - one stable idempotency key per logical activity, reused across retries; - an activity record linking request hash, remote identity, and result evidence; @@ -313,22 +334,31 @@ commits but before the orchestrator records success. Target-v1 handling requires An attempt number is useful audit metadata but is usually the wrong idempotency key: changing it on every retry defeats deduplication. -The alpha runtime has no durable activity ledger or recovery approval gate. -Custom executors are responsible for idempotency today. +The alpha runtime has no durable activity ledger, reconciliation engine, +compensation coordinator, or recovery approval callback. Custom executors and +their applications are responsible for idempotency and ambiguous external +effects. Graph Engineering does not claim external exactly-once execution. ## Durable state and artifact integrity The alpha local adapters implement CAS event append and content-hashed -checkpoint files for one process. They reject unsafe path identifiers and detect -truncated or modified records. They do not provide encryption, tenant -authorization, multi-process locking, leases, scheduler resume, or an artifact -store. Use a private directory owned by the least-privileged runtime identity. +checkpoint files for one process. The native durable schedulers now use event +streams for start/resume and fold the complete history before continuation. +They reject unsafe path identifiers and detect malformed, truncated, or +hash-inconsistent records. They do not provide authentication, encryption, +tenant authorization, multi-process locking, leases, checkpoint-accelerated +resume, or an artifact store. Use a private directory owned by the +least-privileged runtime identity. + +Current recovery binds immutable graph/input/implementation hashes and uses CAS +for every event append. The implementation ID is supplied by the caller and is +not code attestation. CAS detects stale commits but cannot fence an old +coordinator before it performs external work. Stop the prior coordinator before +resuming. The remaining target-v1 requirements are: -- Event appends use expected sequence/version to reject concurrent writers. - One active orchestrator lease owns run progression; lease loss stops scheduling. -- Every run binds an immutable graph hash and implementation/version metadata. - Checkpoints have schema/version and integrity metadata and can be rebuilt from event history. - Artifact references are content-addressed where feasible and include size/media @@ -404,7 +434,9 @@ Before running the current code: - inspect the graph hash and validation diagnostics before execution; - keep local event/checkpoint directories private and do not treat their hashes as signatures or their CAS as a distributed lock; -- assume a process crash requires a fresh run; durable resume is not available; +- resume a crashed durable run only after confirming the old coordinator has + stopped; allow automatic recovery only for truly effect-free or idempotent + activities and reconcile ambiguous external state; - do not expose alpha execution directly to untrusted multi-tenant users. ## Target-v1 security acceptance criteria diff --git a/package.json b/package.json index b39df52..441934a 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,6 @@ "audit:prod": "corepack pnpm audit --prod --audit-level moderate", "check:python-package": "python3 scripts/check-python-artifacts.py", "test:python": "uv run --project python pytest", - "test:conformance": "corepack pnpm --filter @graph-engineering/core build && corepack pnpm --filter @graph-engineering/runtime build && corepack pnpm --filter @graph-engineering/persistence build && corepack pnpm --filter @graph-engineering/primitives build && node tools/conformance/run.mjs" + "test:conformance": "corepack pnpm --filter @graph-engineering/core build && corepack pnpm --filter @graph-engineering/persistence build && corepack pnpm --filter @graph-engineering/runtime build && corepack pnpm --filter @graph-engineering/primitives build && node tools/conformance/run.mjs" } } diff --git a/packages/cli/assets/spec/graph.schema.json b/packages/cli/assets/spec/graph.schema.json index 89aab56..2a9bec7 100644 --- a/packages/cli/assets/spec/graph.schema.json +++ b/packages/cli/assets/spec/graph.schema.json @@ -63,8 +63,8 @@ "additionalProperties": false, "properties": { "maxAttempts": { "type": "integer", "minimum": 1, "maximum": 100 }, - "initialDelayMs": { "type": "integer", "minimum": 0 }, - "maxDelayMs": { "type": "integer", "minimum": 0 }, + "initialDelayMs": { "type": "integer", "minimum": 0, "maximum": 2147483647 }, + "maxDelayMs": { "type": "integer", "minimum": 0, "maximum": 2147483647 }, "backoffMultiplier": { "type": "number", "minimum": 1 }, "jitter": { "type": "boolean" } } @@ -82,7 +82,7 @@ "outputSchema": { "$ref": "#/$defs/jsonSchema" }, "config": {}, "retry": { "$ref": "#/$defs/retry" }, - "timeoutMs": { "type": "integer", "minimum": 1 }, + "timeoutMs": { "type": "integer", "minimum": 1, "maximum": 2147483647 }, "cache": { "type": "object" }, "resources": { "type": "object" }, "isolation": { "type": "object" }, @@ -107,12 +107,12 @@ "type": "object", "additionalProperties": true, "properties": { - "maxConcurrency": { "type": "integer", "minimum": 1 }, - "maxDynamicNodes": { "type": "integer", "minimum": 0 }, - "maxDepth": { "type": "integer", "minimum": 1 }, - "maxFanOut": { "type": "integer", "minimum": 1 }, - "maxTotalAttempts": { "type": "integer", "minimum": 1 }, - "maxDurationMs": { "type": "integer", "minimum": 1 }, + "maxConcurrency": { "type": "integer", "minimum": 1, "maximum": 9007199254740991 }, + "maxDynamicNodes": { "type": "integer", "minimum": 0, "maximum": 9007199254740991 }, + "maxDepth": { "type": "integer", "minimum": 1, "maximum": 9007199254740991 }, + "maxFanOut": { "type": "integer", "minimum": 1, "maximum": 9007199254740991 }, + "maxTotalAttempts": { "type": "integer", "minimum": 1, "maximum": 9007199254740991 }, + "maxDurationMs": { "type": "integer", "minimum": 1, "maximum": 2147483647 }, "maxCostUsd": { "type": "number", "minimum": 0 } } } diff --git a/packages/core/src/schema-validation.ts b/packages/core/src/schema-validation.ts index 335a5d3..0914504 100644 --- a/packages/core/src/schema-validation.ts +++ b/packages/core/src/schema-validation.ts @@ -49,6 +49,7 @@ const EDGE_MODES = new Set(["value", "stream", "artifact-ref"]); const SIDE_EFFECT_MODES = new Set(["none", "idempotent", "non-idempotent"]); const IDENTIFIER = /^[A-Za-z][A-Za-z0-9_.-]{0,127}$/; const GRAPH_NAME = /^[a-z][a-z0-9-]{0,62}$/; +const MAX_TIMER_MILLISECONDS = 2_147_483_647; function record(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); @@ -108,8 +109,10 @@ function validateNode(value: unknown, index: number, issues: string[]): void { if (typeof value.kind !== "string" || !NODE_KINDS.has(value.kind)) issues.push(`${path}/kind: invalid node kind`); if (!record(value.inputSchema)) issues.push(`${path}/inputSchema: expected an object`); if (!record(value.outputSchema)) issues.push(`${path}/outputSchema: expected an object`); - if (value.timeoutMs !== undefined && (!Number.isInteger(value.timeoutMs) || (value.timeoutMs as number) < 1)) { - issues.push(`${path}/timeoutMs: expected a positive integer`); + if (value.timeoutMs !== undefined && + (!Number.isSafeInteger(value.timeoutMs) || (value.timeoutMs as number) < 1 || + (value.timeoutMs as number) > MAX_TIMER_MILLISECONDS)) { + issues.push(`${path}/timeoutMs: expected an integer from 1 to ${MAX_TIMER_MILLISECONDS}`); } if (value.sideEffects !== undefined && (typeof value.sideEffects !== "string" || !SIDE_EFFECT_MODES.has(value.sideEffects))) { issues.push(`${path}/sideEffects: invalid mode`); @@ -123,12 +126,16 @@ function validateNode(value: unknown, index: number, issues: string[]): void { } else { unknownKeys(value.retry, RETRY_KEYS, `${path}/retry`, issues); const { maxAttempts, initialDelayMs, maxDelayMs, backoffMultiplier, jitter } = value.retry; - if (maxAttempts !== undefined && (!Number.isInteger(maxAttempts) || (maxAttempts as number) < 1 || (maxAttempts as number) > 100)) { + if (maxAttempts !== undefined && (!Number.isSafeInteger(maxAttempts) || (maxAttempts as number) < 1 || (maxAttempts as number) > 100)) { issues.push(`${path}/retry/maxAttempts: expected an integer from 1 to 100`); } for (const [name, delayValue] of [["initialDelayMs", initialDelayMs], ["maxDelayMs", maxDelayMs]] as const) { - if (delayValue !== undefined && (!Number.isInteger(delayValue) || (delayValue as number) < 0)) { - issues.push(`${path}/retry/${name}: expected a non-negative integer`); + if (delayValue !== undefined && + (!Number.isSafeInteger(delayValue) || (delayValue as number) < 0 || + (delayValue as number) > MAX_TIMER_MILLISECONDS)) { + issues.push( + `${path}/retry/${name}: expected an integer from 0 to ${MAX_TIMER_MILLISECONDS}`, + ); } } if (backoffMultiplier !== undefined && (typeof backoffMultiplier !== "number" || backoffMultiplier < 1)) { @@ -204,16 +211,23 @@ export function validateGraphDocument(value: unknown): readonly string[] { "maxDepth", "maxFanOut", "maxTotalAttempts", - "maxDurationMs", ] as const; for (const key of positiveIntegers) { const item = value.policies[key]; - if (item !== undefined && (!Number.isInteger(item) || (item as number) < 1)) { + if (item !== undefined && (!Number.isSafeInteger(item) || (item as number) < 1)) { issues.push(`#/policies/${key}: expected a positive integer`); } } + const maxDurationMs = value.policies.maxDurationMs; + if (maxDurationMs !== undefined && + (!Number.isSafeInteger(maxDurationMs) || (maxDurationMs as number) < 1 || + (maxDurationMs as number) > MAX_TIMER_MILLISECONDS)) { + issues.push( + `#/policies/maxDurationMs: expected an integer from 1 to ${MAX_TIMER_MILLISECONDS}`, + ); + } const dynamicNodes = value.policies.maxDynamicNodes; - if (dynamicNodes !== undefined && (!Number.isInteger(dynamicNodes) || (dynamicNodes as number) < 0)) { + if (dynamicNodes !== undefined && (!Number.isSafeInteger(dynamicNodes) || (dynamicNodes as number) < 0)) { issues.push("#/policies/maxDynamicNodes: expected a non-negative integer"); } const cost = value.policies.maxCostUsd; diff --git a/packages/core/test/compiler.test.ts b/packages/core/test/compiler.test.ts index bb6d5c6..0abc0b8 100644 --- a/packages/core/test/compiler.test.ts +++ b/packages/core/test/compiler.test.ts @@ -89,6 +89,26 @@ describe("graph compiler conformance", () => { ]); }); + it("accepts the single-timer ceiling and rejects every oversized timer field", () => { + const maximum = 2_147_483_647; + expect(compileGraph(graph({ + nodes: [node("a", { + timeoutMs: maximum, + retry: { maxAttempts: 2, initialDelayMs: maximum, maxDelayMs: maximum }, + })], + policies: { maxDurationMs: maximum }, + })).valid).toBe(true); + + const oversized = compileGraph(fixture("invalid-oversized-timers.graph.json")); + expect(oversized).toMatchObject({ + valid: false, + graphHash: null, + canonicalGraph: null, + diagnostics: [expect.objectContaining({ code: "GE1007_INVALID_GRAPH" })], + }); + expect(oversized.diagnostics[0]?.message).toMatch(/timeoutMs.*initialDelayMs.*maxDelayMs.*maxDurationMs/); + }); + it("orders ready peers by semantic node declaration order", () => { const result = compileGraph( graph({ diff --git a/packages/mcp-server/schemas/v1alpha1/graph.schema.json b/packages/mcp-server/schemas/v1alpha1/graph.schema.json index 89aab56..2a9bec7 100644 --- a/packages/mcp-server/schemas/v1alpha1/graph.schema.json +++ b/packages/mcp-server/schemas/v1alpha1/graph.schema.json @@ -63,8 +63,8 @@ "additionalProperties": false, "properties": { "maxAttempts": { "type": "integer", "minimum": 1, "maximum": 100 }, - "initialDelayMs": { "type": "integer", "minimum": 0 }, - "maxDelayMs": { "type": "integer", "minimum": 0 }, + "initialDelayMs": { "type": "integer", "minimum": 0, "maximum": 2147483647 }, + "maxDelayMs": { "type": "integer", "minimum": 0, "maximum": 2147483647 }, "backoffMultiplier": { "type": "number", "minimum": 1 }, "jitter": { "type": "boolean" } } @@ -82,7 +82,7 @@ "outputSchema": { "$ref": "#/$defs/jsonSchema" }, "config": {}, "retry": { "$ref": "#/$defs/retry" }, - "timeoutMs": { "type": "integer", "minimum": 1 }, + "timeoutMs": { "type": "integer", "minimum": 1, "maximum": 2147483647 }, "cache": { "type": "object" }, "resources": { "type": "object" }, "isolation": { "type": "object" }, @@ -107,12 +107,12 @@ "type": "object", "additionalProperties": true, "properties": { - "maxConcurrency": { "type": "integer", "minimum": 1 }, - "maxDynamicNodes": { "type": "integer", "minimum": 0 }, - "maxDepth": { "type": "integer", "minimum": 1 }, - "maxFanOut": { "type": "integer", "minimum": 1 }, - "maxTotalAttempts": { "type": "integer", "minimum": 1 }, - "maxDurationMs": { "type": "integer", "minimum": 1 }, + "maxConcurrency": { "type": "integer", "minimum": 1, "maximum": 9007199254740991 }, + "maxDynamicNodes": { "type": "integer", "minimum": 0, "maximum": 9007199254740991 }, + "maxDepth": { "type": "integer", "minimum": 1, "maximum": 9007199254740991 }, + "maxFanOut": { "type": "integer", "minimum": 1, "maximum": 9007199254740991 }, + "maxTotalAttempts": { "type": "integer", "minimum": 1, "maximum": 9007199254740991 }, + "maxDurationMs": { "type": "integer", "minimum": 1, "maximum": 2147483647 }, "maxCostUsd": { "type": "number", "minimum": 0 } } } diff --git a/packages/persistence/src/events.ts b/packages/persistence/src/events.ts index 9dd74ac..45ba69c 100644 --- a/packages/persistence/src/events.ts +++ b/packages/persistence/src/events.ts @@ -14,6 +14,7 @@ export const GRAPH_EVENT_TYPES = [ "NodeScheduled", "NodeStarted", "NodeAttemptFailed", + "NodeSettledWithoutAttempt", "NodeRetried", "NodeSucceeded", "EdgeEmitted", diff --git a/packages/runtime/README.md b/packages/runtime/README.md index fe39c42..03ea002 100644 --- a/packages/runtime/README.md +++ b/packages/runtime/README.md @@ -1,7 +1,9 @@ # `@graph-engineering/runtime` -A small, deterministic TypeScript scheduler for the Graph Engineering -v1alpha1 IR. +A small, deterministic TypeScript scheduler for the Graph Engineering v1alpha1 +IR, with both in-memory execution and event-sourced durable continuation. + +## In-memory execution ```ts import { runGraph } from "@graph-engineering/runtime"; @@ -15,6 +17,64 @@ const result = await runGraph(graph, { query: "graph engineering" }, { }); ``` +`runGraph` does not persist progress. Use the separate durable operations when a +run must continue from committed scheduler history after process loss. + +## Durable start and resume + +```ts +import { JsonlEventStore } from "@graph-engineering/persistence"; +import { + resumeDurableGraphRun, + startDurableGraphRun, +} from "@graph-engineering/runtime"; + +const eventStore = new JsonlEventStore({ directory: ".graph-engineering" }); +const options = { + runId: "research-001", + implementationId: "research-handlers@1", + eventStore, + nodeExecutors: { + research: async ({ input, signal, idempotencyKey }) => + search(input, { signal, idempotencyKey }), + synthesize: async ({ input }) => writeReport(input), + }, + concurrency: 8, +}; + +// Invoke with --resume only in a replacement process after confirming that the +// old coordinator stopped. Resume throws RUN_NOT_FOUND for a missing run and +// never accepts replacement input; start throws RUN_ALREADY_EXISTS instead of +// silently resuming. +const result = process.argv.includes("--resume") + ? await resumeDurableGraphRun(graph, options) + : await startDurableGraphRun( + graph, + { query: "graph engineering" }, + options, + ); +``` + +The event stream is authoritative. A durable attempt claim commits before its +executor is called; a validated success and ordered edge emissions commit before +dependants are released. Resume verifies the bound graph, original input, and +caller-supplied `implementationId`, then reuses committed successful nodes. + +An open attempt has an unknown outcome. Nodes declared +`sideEffects: "none"` or `"idempotent"` may retry within their original node and +global budgets. Idempotent attempts receive the same `activityKey` and +`idempotencyKey`, which the executor must forward to the external system. An +omitted or `"non-idempotent"` declaration fails closed with +`IN_DOUBT_SIDE_EFFECT` and is not invoked again. A valid terminal resume returns +the recorded result with zero new events and zero executor calls. + +This alpha recovery path folds the complete event history. It has no scheduler +checkpoint acceleration, distributed lease/fencing, replay/fork, external +exactly-once guarantee, durable activity ledger, or approval callback. The local +JSONL store coordinates one process; confirm that the former coordinator has +stopped before resume. See the +[durable recovery semantics](../../spec/durable-recovery-semantics.md). + ## Alpha semantics - ready nodes execute concurrently up to the graph policy and caller limit; @@ -35,6 +95,7 @@ const result = await runGraph(graph, { query: "graph engineering" }, { configured bounded retry policy. Finite non-integer doubles remain valid; - transform and barrier nodes default to deterministic identity executors. -Edge `condition`/`map`, JSON Schema I/O validation, streaming edges, durable -checkpoints, and distributed workers are intentionally scheduled for later -alphas. They are not silently emulated in this package. +Edge `condition`/`map`, JSON Schema I/O validation, streaming edges, scheduler +checkpoint acceleration, distributed workers, and distributed leases are +intentionally scheduled for later alphas. They are not silently emulated in this +package. diff --git a/packages/runtime/package.json b/packages/runtime/package.json index 210891d..f9f8c7e 100644 --- a/packages/runtime/package.json +++ b/packages/runtime/package.json @@ -36,7 +36,8 @@ "prepack": "npm run build" }, "dependencies": { - "@graph-engineering/core": "workspace:*" + "@graph-engineering/core": "workspace:*", + "@graph-engineering/persistence": "workspace:*" }, "devDependencies": { "@types/node": "^22.0.0", diff --git a/packages/runtime/src/durable-json.ts b/packages/runtime/src/durable-json.ts new file mode 100644 index 0000000..4c94781 --- /dev/null +++ b/packages/runtime/src/durable-json.ts @@ -0,0 +1,155 @@ +import { createHash } from "node:crypto"; +import { compareUnicodeCodePoints } from "@graph-engineering/core"; +import { snapshotJson } from "./json.js"; +import type { JsonValue } from "./types.js"; + +export type DurableJson = + | readonly ["n"] + | readonly ["b", boolean] + | readonly ["s", string] + | readonly ["i", number] + | readonly ["f", string] + | readonly ["a", readonly DurableJson[]] + | readonly ["o", readonly (readonly [string, DurableJson])[]]; + +function freeze(value: T): T { + for (const child of value) { + if (Array.isArray(child)) freeze(child); + } + return Object.freeze(value); +} + +function floatBits(value: number): string { + const bytes = new Uint8Array(8); + new DataView(bytes.buffer).setFloat64(0, value, false); + return [...bytes].map((byte) => byte.toString(16).padStart(2, "0")).join(""); +} + +function bitsFloat(value: string): number { + const bytes = new Uint8Array(8); + for (let index = 0; index < 8; index += 1) { + bytes[index] = Number.parseInt(value.slice(index * 2, index * 2 + 2), 16); + } + return new DataView(bytes.buffer).getFloat64(0, false); +} + +function encode(value: JsonValue): DurableJson { + if (value === null) return freeze(["n"] as const); + if (typeof value === "boolean") return freeze(["b", value] as const); + if (typeof value === "string") return freeze(["s", value] as const); + if (typeof value === "number") { + if (Number.isInteger(value) || Object.is(value, -0)) { + return freeze(["i", value === 0 ? 0 : value] as const); + } + return freeze(["f", floatBits(value)] as const); + } + if (Array.isArray(value)) { + return freeze(["a", Object.freeze(value.map(encode))] as const); + } + const record = value as Readonly>; + const entries = Object.keys(record) + .sort(compareUnicodeCodePoints) + .map((key) => freeze([key, encode(record[key] as JsonValue)] as const)); + return freeze(["o", Object.freeze(entries)] as const); +} + +/** Encode a detached portable runtime JSON value without losing binary64 bits. */ +export function encodeDurableJson(value: unknown): DurableJson { + return encode(snapshotJson(value)); +} + +function invalid(message: string): never { + throw new TypeError(`Invalid Durable JSON: ${message}`); +} + +function exactArray(value: JsonValue, length: number, tag?: string): readonly JsonValue[] { + if (!Array.isArray(value) || value.length !== length) { + return invalid(tag === undefined ? "expected a tagged array" : `tag '${tag}' has invalid arity`); + } + return value; +} + +function decode(value: JsonValue): JsonValue { + const tuple = exactArray(value, Array.isArray(value) ? value.length : 0); + const tag = tuple[0]; + if (typeof tag !== "string" || tag.length !== 1) return invalid("unknown tag"); + + switch (tag) { + case "n": + exactArray(tuple, 1, tag); + return null; + case "b": { + const tagged = exactArray(tuple, 2, tag); + if (typeof tagged[1] !== "boolean") return invalid("tag 'b' requires a boolean"); + return tagged[1]; + } + case "s": { + const tagged = exactArray(tuple, 2, tag); + if (typeof tagged[1] !== "string") return invalid("tag 's' requires a string"); + return tagged[1]; + } + case "i": { + const tagged = exactArray(tuple, 2, tag); + if (typeof tagged[1] !== "number" || !Number.isSafeInteger(tagged[1])) { + return invalid("tag 'i' requires a safe integer"); + } + return tagged[1] === 0 ? 0 : tagged[1]; + } + case "f": { + const tagged = exactArray(tuple, 2, tag); + if (typeof tagged[1] !== "string" || !/^[a-f0-9]{16}$/.test(tagged[1])) { + return invalid("tag 'f' requires sixteen lowercase hexadecimal digits"); + } + const number = bitsFloat(tagged[1]); + if (!Number.isFinite(number) || Number.isInteger(number) || Object.is(number, -0)) { + return invalid("tag 'f' must encode a finite non-integer binary64 value"); + } + return number; + } + case "a": { + const tagged = exactArray(tuple, 2, tag); + if (!Array.isArray(tagged[1])) return invalid("tag 'a' requires an item array"); + return Object.freeze(tagged[1].map(decode)); + } + case "o": { + const tagged = exactArray(tuple, 2, tag); + if (!Array.isArray(tagged[1])) return invalid("tag 'o' requires an entry array"); + const result = Object.create(null) as Record; + let previous: string | undefined; + for (const entry of tagged[1]) { + if (!Array.isArray(entry) || entry.length !== 2 || typeof entry[0] !== "string") { + return invalid("tag 'o' entries must be [string, tagged-value] pairs"); + } + if (previous !== undefined && compareUnicodeCodePoints(previous, entry[0]) >= 0) { + return invalid("tag 'o' keys must be unique and sorted by Unicode code point"); + } + result[entry[0]] = decode(entry[1] as JsonValue); + previous = entry[0]; + } + return Object.freeze(result); + } + default: + return invalid(`unknown tag '${tag}'`); + } +} + +/** Decode and strictly validate a canonical tagged Durable JSON value. */ +export function decodeDurableJson(value: unknown): JsonValue { + const snapshot = snapshotJson(value); + const decoded = decode(snapshot); + const canonical = encode(decoded); + if (JSON.stringify(canonical) !== JSON.stringify(snapshot)) { + return invalid("value is not in canonical tagged form"); + } + // The decoder builds objects with a null prototype so assigning an own + // `__proto__` key cannot mutate a prototype. Re-snapshot only after the + // canonical tagged form has been proven; callers then receive the same + // deeply frozen, ordinary JSON object shape as every other runtime boundary. + return snapshotJson(decoded); +} + +/** SHA-256 of canonical UTF-8 JSON for the tagged representation of a value. */ +export function durableJsonHash(value: unknown): string { + const tagged = encodeDurableJson(value); + return createHash("sha256").update(JSON.stringify(tagged), "utf8").digest("hex"); +} diff --git a/packages/runtime/src/durable-types.ts b/packages/runtime/src/durable-types.ts new file mode 100644 index 0000000..63e84c8 --- /dev/null +++ b/packages/runtime/src/durable-types.ts @@ -0,0 +1,85 @@ +import type { GraphSpec, NodeKind, NodeSpec } from "@graph-engineering/core"; +import type { EventStore } from "@graph-engineering/persistence"; +import type { GraphRunResult, JsonValue } from "./types.js"; + +export type DurableRunErrorCode = + | "RUN_NOT_FOUND" + | "RUN_ALREADY_EXISTS" + | "GRAPH_HASH_MISMATCH" + | "INPUT_HASH_MISMATCH" + | "IMPLEMENTATION_MISMATCH" + | "INVALID_RUN_HISTORY" + | "NODE_EXECUTION_INTERRUPTED" + | "IN_DOUBT_SIDE_EFFECT" + | "RESUME_CONFLICT" + | "DURABILITY_STORE_FAILED"; + +export interface SerializedDurableRunError { + name: "DurableRunError"; + code: DurableRunErrorCode; + runId: string; + message: string; + details: Readonly>; +} + +export class DurableRunError extends Error { + readonly code: DurableRunErrorCode; + readonly runId: string; + readonly details: Readonly>; + + constructor( + code: DurableRunErrorCode, + runId: string, + message: string, + details: Readonly> = {}, + options: ErrorOptions = {}, + ) { + super(message, options); + this.name = "DurableRunError"; + this.code = code; + this.runId = runId; + this.details = details; + } + + toJSON(): SerializedDurableRunError { + return { name: "DurableRunError", code: this.code, runId: this.runId, message: this.message, details: this.details }; + } +} + +export interface DurableNodeExecutionContext { + graph: GraphSpec; + node: NodeSpec; + input: JsonValue; + attempt: number; + signal: AbortSignal; + runId: string; + attemptId: string; + /** Stable across attempts for the same logical node activity. */ + activityKey: string; + /** Alias intended for external idempotency APIs. */ + idempotencyKey: string; +} + +export type DurableNodeExecutor = ( + context: DurableNodeExecutionContext, +) => unknown | Promise; + +export interface DurableEventIdContext { + runId: string; + sequence: number; + type: string; +} + +export interface DurableSchedulerOptions { + runId: string; + implementationId: string; + eventStore: EventStore; + nodeExecutors?: Readonly>; + executors?: Readonly>>; + concurrency?: number; + signal?: AbortSignal; + now?: () => Date; + createEventId?: (context: DurableEventIdContext) => string; +} + +export type DurableGraphRunResult = GraphRunResult; diff --git a/packages/runtime/src/durable.ts b/packages/runtime/src/durable.ts new file mode 100644 index 0000000..b3b95ef --- /dev/null +++ b/packages/runtime/src/durable.ts @@ -0,0 +1,2139 @@ +import { + canonicalHash, + compareUnicodeCodePoints, + compileGraph, + type EdgeSpec, + type Endpoint, + type GraphSpec, + type NodeSpec, +} from "@graph-engineering/core"; +import { + GRAPH_EVENT_API_VERSION, + PersistenceError, + VersionConflictError, + assertGraphEvent, + type EventStore, + type GraphEvent, + type GraphEventType, +} from "@graph-engineering/persistence"; +import { + type SchedulerAttemptIdentity, + type SchedulerInternalOptions, + type SchedulerJournal, + type SchedulerRunResult, + runGraphWithJournal, +} from "./scheduler.js"; +import { decodeDurableJson, durableJsonHash, encodeDurableJson } from "./durable-json.js"; +import { + DurableRunError, + type DurableGraphRunResult, + type DurableNodeExecutionContext, + type DurableNodeExecutor, + type DurableRunErrorCode, + type DurableSchedulerOptions, +} from "./durable-types.js"; +import { snapshotJson } from "./json.js"; +import type { + GraphRunFailure, + GraphRunResult, + JsonValue, + NodeRunFailure, + NodeRunResult, + SchedulerOptions, +} from "./types.js"; + +const CONTRACT_VERSION = "scheduler-recovery/v1alpha1"; +const GRAPH_REVISION = 1; +const EVENT_API_VERSION = GRAPH_EVENT_API_VERSION; +const MAX_TIMER_DELAY_MS = 2_147_483_647; + +type JsonRecord = Record; + +interface CompiledDurableGraph { + readonly graph: GraphSpec; + readonly graphHash: string; + readonly orderedNodeIds: readonly string[]; + readonly nodesById: ReadonlyMap; + readonly incoming: ReadonlyMap; + readonly outgoing: ReadonlyMap; + readonly sequence: ReadonlyMap; + readonly declaration: ReadonlyMap; +} + +interface EventDraft { + readonly type: GraphEventType; + readonly data: Readonly; + readonly nodeId?: string; + readonly edgeId?: string; + readonly attempt?: number; +} + +function invalidHistory( + runId: string, + message: string, + details: Readonly> = {}, + options: ErrorOptions = {}, +): DurableRunError { + return new DurableRunError("INVALID_RUN_HISTORY", runId, message, details, options); +} + +function errorName(error: unknown): string { + return error instanceof Error ? error.name : typeof error; +} + +function errorMessage(error: unknown): string { + if (error instanceof Error) return error.message; + return typeof error === "string" ? error : "unknown durable scheduler failure"; +} + +function compileDurableGraph(document: GraphSpec): CompiledDurableGraph | GraphRunResult { + const compilation = compileGraph(document); + if (!compilation.valid || compilation.graphHash === null || compilation.canonicalGraph === null) { + const failures: GraphRunFailure[] = compilation.diagnostics + .filter((item) => item.severity === "error") + .map((item) => ({ + phase: "compile", + code: item.code, + message: item.message, + diagnostic: item, + })); + return { + status: "failed", + graphHash: compilation.graphHash, + nodes: [], + failures, + maxObservedConcurrency: 0, + totalAttempts: 0, + }; + } + + const graph = snapshotJson(JSON.parse(compilation.canonicalGraph)) as unknown as GraphSpec; + const orderedNodeIds = compilation.topologicalLayers.flatMap((layer) => layer); + const nodesById = new Map(graph.nodes.map((node) => [node.id, node])); + const incoming = new Map(graph.nodes.map((node) => [node.id, [] as EdgeSpec[]])); + const outgoing = new Map(graph.nodes.map((node) => [node.id, [] as EdgeSpec[]])); + for (const edge of graph.edges) { + incoming.get(edge.to.node)?.push(edge); + outgoing.get(edge.from.node)?.push(edge); + } + for (const edges of incoming.values()) { + edges.sort((left, right) => compareUnicodeCodePoints(left.id, right.id)); + } + for (const edges of outgoing.values()) { + edges.sort((left, right) => compareUnicodeCodePoints(left.id, right.id)); + } + return { + graph, + graphHash: compilation.graphHash, + orderedNodeIds, + nodesById, + incoming, + outgoing, + sequence: new Map(orderedNodeIds.map((nodeId, index) => [nodeId, index])), + declaration: new Map(graph.nodes.map((node, index) => [node.id, index])), + }; +} + +function isCompilationFailure( + value: CompiledDurableGraph | GraphRunResult, +): value is GraphRunResult { + return "status" in value; +} + +function sideEffects(node: NodeSpec): "none" | "idempotent" | "non-idempotent" | "unspecified" { + return node.sideEffects ?? "unspecified"; +} + +function effectiveAttemptLimit(graph: GraphSpec): number { + return graph.policies?.maxTotalAttempts ?? Number.MAX_SAFE_INTEGER; +} + +function assertDurableTimerBounds(graph: GraphSpec): void { + for (const node of graph.nodes) { + for (const [name, value] of [ + ["timeoutMs", node.timeoutMs], + ["retry.initialDelayMs", node.retry?.initialDelayMs], + ["retry.maxDelayMs", node.retry?.maxDelayMs], + ] as const) { + if (value !== undefined && value > MAX_TIMER_DELAY_MS) { + throw new TypeError( + `Node '${node.id}' ${name} exceeds the ${MAX_TIMER_DELAY_MS}ms durable timer limit`, + ); + } + } + } + const maxDurationMs = graph.policies?.maxDurationMs; + if (typeof maxDurationMs === "number" && maxDurationMs > MAX_TIMER_DELAY_MS) { + throw new TypeError(`maxDurationMs exceeds the ${MAX_TIMER_DELAY_MS}ms durable timer limit`); + } +} + +function retryDelayMs(node: NodeSpec, failedAttempt: number): number { + const initial = Math.max(0, node.retry?.initialDelayMs ?? 0); + const multiplier = Math.max(1, node.retry?.backoffMultiplier ?? 1); + const maximum = Math.max(initial, node.retry?.maxDelayMs ?? initial); + return Math.min(maximum, initial * multiplier ** Math.max(0, failedAttempt - 1)); +} + +function strictDate(value: unknown, runId: string, context: string): Date { + const match = typeof value === "string" + ? /^(\d{4})-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01])T([01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|[+-](?:[01]\d|2[0-3]):[0-5]\d)$/.exec(value) + : null; + if (match === null) { + throw invalidHistory(runId, `${context} must be a strict RFC 3339 timestamp`); + } + const year = Number(match[1]); + const month = Number(match[2]); + const day = Number(match[3]); + const leap = year % 4 === 0 && (year % 100 !== 0 || year % 400 === 0); + const daysInMonth = [31, leap ? 29 : 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]; + if (year < 1 || day > (daysInMonth[month - 1] as number)) { + throw invalidHistory(runId, `${context} contains an invalid calendar date`); + } + const parsed = new Date(value as string); + if (!Number.isFinite(parsed.getTime())) { + throw invalidHistory(runId, `${context} is not a real timestamp`); + } + return parsed; +} + +function addMilliseconds(date: Date, milliseconds: number): string { + const result = new Date(date.getTime() + milliseconds); + const year = result.getUTCFullYear(); + if (!Number.isFinite(result.getTime()) || year < 1 || year > 9999) { + throw new RangeError("retry availability is outside four-digit RFC 3339 range"); + } + return result.toISOString(); +} + +function failureDocument(failure: NodeRunFailure): JsonRecord { + return { + phase: failure.phase, + code: failure.code, + message: failure.message, + nodeId: failure.nodeId, + attempt: failure.attempt, + retryable: failure.retryable, + ...(failure.causeName === undefined ? {} : { causeName: failure.causeName }), + ...(failure.upstreamNodeIds === undefined + ? {} + : { upstreamNodeIds: [...failure.upstreamNodeIds] }), + }; +} + +function graphFailureDocument(failure: GraphRunFailure): JsonRecord { + if (failure.phase === "execute") return failureDocument(failure); + if (failure.phase === "output") { + return { + phase: "output", + code: failure.code, + message: failure.message, + outputName: failure.outputName, + nodeId: failure.nodeId, + ...(failure.port === undefined ? {} : { port: failure.port }), + }; + } + return { + phase: "compile", + code: failure.code, + message: failure.message, + diagnostic: snapshotJson(failure.diagnostic), + }; +} + +function nodeResultDocument(result: NodeRunResult): JsonRecord { + return { + nodeId: result.nodeId, + sequence: result.sequence, + status: result.status, + attempts: result.attempts, + ...(result.input === undefined ? {} : { input: result.input }), + ...(result.output === undefined ? {} : { output: result.output }), + ...(result.failure === undefined ? {} : { failure: failureDocument(result.failure) }), + }; +} + +function resultDocument(result: SchedulerRunResult): JsonRecord { + return snapshotJson({ + status: result.status, + graphHash: result.graphHash, + nodes: result.nodes.map(nodeResultDocument), + failures: result.failures.map(graphFailureDocument), + scheduledOrder: [...result.scheduledOrder], + completionOrder: [...result.completionOrder], + maxObservedConcurrency: result.maxObservedConcurrency, + totalAttempts: result.totalAttempts, + ...(result.output === undefined ? {} : { output: result.output }), + }) as JsonRecord; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function exactKeys( + value: unknown, + keys: readonly string[], + runId: string, + context: string, +): asserts value is Record { + if (!isRecord(value)) throw invalidHistory(runId, `${context} must be an object`); + const actual = Object.keys(value).sort(compareUnicodeCodePoints); + const expected = [...keys].sort(compareUnicodeCodePoints); + if (JSON.stringify(actual) !== JSON.stringify(expected)) { + throw invalidHistory(runId, `${context} has unexpected fields`, { expected, actual }); + } +} + +function stringValue(value: unknown, runId: string, context: string): string { + if (typeof value !== "string") throw invalidHistory(runId, `${context} must be a string`); + return value; +} + +function integerValue( + value: unknown, + runId: string, + context: string, + minimum = 0, +): number { + if (!Number.isSafeInteger(value) || (value as number) < minimum) { + throw invalidHistory(runId, `${context} must be a safe integer >= ${minimum}`); + } + return value as number; +} + +function arrayValue(value: unknown, runId: string, context: string): readonly unknown[] { + if (!Array.isArray(value)) throw invalidHistory(runId, `${context} must be an array`); + return value; +} + +function endpointValue(endpoint: Endpoint, value: JsonValue | undefined): JsonValue { + if (endpoint.port === undefined) return value as JsonValue; + if (typeof value !== "object" || value === null || !Object.hasOwn(value, endpoint.port)) { + throw new Error(`Output from '${endpoint.node}' does not contain port '${endpoint.port}'`); + } + return (value as Readonly>)[endpoint.port] as JsonValue; +} + +function defineJsonValue(target: Record, key: string, value: JsonValue): void { + Object.defineProperty(target, key, { + value, + enumerable: true, + configurable: true, + writable: true, + }); +} + +function bindInput( + compiled: CompiledDurableGraph, + nodeId: string, + graphInput: JsonValue, + results: ReadonlyMap, +): JsonValue { + const incoming = compiled.incoming.get(nodeId) ?? []; + if (incoming.length === 0) return snapshotJson(graphInput); + const input = Object.create(null) as Record; + for (const edge of incoming) { + const producer = results.get(edge.from.node); + if (producer?.status !== "succeeded") { + throw new Error(`Node '${nodeId}' is not ready because '${edge.from.node}' has not succeeded`); + } + const key = edge.to.port ?? edge.from.node; + if (Object.hasOwn(input, key)) { + throw new Error(`Node '${nodeId}' receives duplicate input key '${key}'`); + } + defineJsonValue(input, key, endpointValue(edge.from, producer.output)); + } + return snapshotJson(input); +} + +class DurableJournal implements SchedulerJournal { + readonly compiled: CompiledDurableGraph; + readonly runId: string; + readonly store: EventStore; + readonly now: () => Date; + readonly createEventId: NonNullable; + readonly preScheduled: Map; + readonly retryAvailableAt = new Map(); + readonly activityKeys: Map; + readonly eventIds: Set; + + version: number; + #queue: Promise = Promise.resolve(); + #fatal: unknown; + + constructor(fields: { + compiled: CompiledDurableGraph; + runId: string; + store: EventStore; + version: number; + now: () => Date; + createEventId: NonNullable; + preScheduled?: ReadonlyMap; + activityKeys?: ReadonlyMap; + eventIds?: ReadonlySet; + }) { + this.compiled = fields.compiled; + this.runId = fields.runId; + this.store = fields.store; + this.version = fields.version; + this.now = fields.now; + this.createEventId = fields.createEventId; + this.preScheduled = new Map(fields.preScheduled); + this.activityKeys = new Map(fields.activityKeys); + this.eventIds = new Set(fields.eventIds); + } + + activityKey(nodeId: string, inputHash: string): string { + return durableJsonHash([ + "activity/v1alpha1", + this.runId, + GRAPH_REVISION, + nodeId, + inputHash, + ]); + } + + #event(draft: EventDraft, sequence: number): GraphEvent { + const document: GraphEvent = { + apiVersion: EVENT_API_VERSION, + eventId: this.createEventId({ runId: this.runId, sequence, type: draft.type }), + type: draft.type, + timestamp: this.now().toISOString(), + runId: this.runId, + graphRevision: GRAPH_REVISION, + sequence, + payloadHash: canonicalHash(draft.data), + redacted: true, + data: draft.data, + ...(draft.nodeId === undefined ? {} : { nodeId: draft.nodeId }), + ...(draft.edgeId === undefined ? {} : { edgeId: draft.edgeId }), + ...(draft.attempt === undefined ? {} : { attempt: draft.attempt }), + }; + assertGraphEvent(document); + return document; + } + + append( + drafts: readonly EventDraft[], + conflictCode: DurableRunErrorCode = "RESUME_CONFLICT", + ): Promise { + const operation = this.#queue.then(async () => { + if (this.#fatal !== undefined) throw this.#fatal; + const expectedVersion = this.version; + try { + const events = drafts.map((draft, index) => + this.#event(draft, expectedVersion + index + 1), + ); + const batchIds = new Set(); + for (const event of events) { + if (this.eventIds.has(event.eventId) || batchIds.has(event.eventId)) { + throw new DurableRunError( + "DURABILITY_STORE_FAILED", + this.runId, + "event ID factory produced a duplicate identifier", + { eventId: event.eventId, sequence: event.sequence }, + ); + } + batchIds.add(event.eventId); + } + const actual = await this.store.append(this.runId, expectedVersion, events); + const expected = expectedVersion + events.length; + if (actual !== expected) { + throw new DurableRunError( + "DURABILITY_STORE_FAILED", + this.runId, + "event store returned an invalid last sequence", + { expected, actual }, + ); + } + this.version = actual; + for (const eventId of batchIds) this.eventIds.add(eventId); + return events; + } catch (error) { + let mapped: DurableRunError; + if (error instanceof DurableRunError) { + mapped = error; + } else if (error instanceof VersionConflictError) { + mapped = new DurableRunError( + conflictCode, + this.runId, + "durable append lost its expected-version comparison", + error.details, + { cause: error }, + ); + } else if (error instanceof PersistenceError) { + mapped = new DurableRunError( + "DURABILITY_STORE_FAILED", + this.runId, + "durable event append failed", + { persistenceCode: error.code, ...error.details }, + { cause: error }, + ); + } else { + mapped = new DurableRunError( + "DURABILITY_STORE_FAILED", + this.runId, + "durable event append failed", + { causeName: errorName(error) }, + { cause: error }, + ); + } + this.#fatal = mapped; + throw mapped; + } + }); + this.#queue = operation.then( + () => undefined, + () => undefined, + ); + return operation; + } + + async beforeAttempt(fields: { + graph: GraphSpec; + node: NodeSpec; + input: JsonValue; + attempt: number; + signal: AbortSignal; + }): Promise { + const input = snapshotJson(fields.input); + const inputHash = durableJsonHash(input); + const activityKey = this.activityKey(fields.node.id, inputHash); + this.activityKeys.set(fields.node.id, activityKey); + const drafts: EventDraft[] = []; + if (this.preScheduled.get(fields.node.id) !== fields.attempt) { + drafts.push({ + type: "NodeScheduled", + nodeId: fields.node.id, + attempt: fields.attempt, + data: { + input: encodeDurableJson(input), + inputHash, + activityKey, + sideEffects: sideEffects(fields.node), + }, + }); + } + this.preScheduled.delete(fields.node.id); + drafts.push({ + type: "NodeStarted", + nodeId: fields.node.id, + attempt: fields.attempt, + data: { inputHash, activityKey }, + }); + await this.append(drafts); + return { + runId: this.runId, + attemptId: `${this.runId}/${fields.node.id}/${fields.attempt}`, + activityKey, + }; + } + + async attemptFailed(fields: { + graph: GraphSpec; + node: NodeSpec; + input: JsonValue; + failure: NodeRunFailure; + identity: SchedulerAttemptIdentity; + willRetry: boolean; + retryDelayMs: number; + }): Promise { + const drafts: EventDraft[] = [{ + type: "NodeAttemptFailed", + nodeId: fields.node.id, + attempt: fields.failure.attempt, + data: { + terminal: !fields.willRetry, + failure: failureDocument(fields.failure), + }, + }]; + if (fields.willRetry) { + const activityKey = this.activityKeys.get(fields.node.id) ?? fields.identity.activityKey; + if (activityKey.length === 0) { + throw invalidHistory(this.runId, "retry lacks a stable activity key", { + nodeId: fields.node.id, + attempt: fields.failure.attempt, + }); + } + let availableAt: string; + try { + if (this.#fatal !== undefined) throw this.#fatal; + availableAt = addMilliseconds(this.now(), fields.retryDelayMs); + } catch (error) { + const mapped = error instanceof DurableRunError + ? error + : new DurableRunError( + "DURABILITY_STORE_FAILED", + this.runId, + "retry availability could not be represented as durable RFC 3339 time", + { causeName: errorName(error), retryDelayMs: fields.retryDelayMs }, + { cause: error }, + ); + this.#fatal = mapped; + throw mapped; + } + this.retryAvailableAt.set(fields.node.id, availableAt); + drafts.push({ + type: "NodeRetried", + nodeId: fields.node.id, + attempt: fields.failure.attempt + 1, + data: { availableAt, activityKey }, + }); + } + await this.append(drafts); + } + + async nodeSucceeded(fields: { + graph: GraphSpec; + node: NodeSpec; + input: JsonValue; + attempt: number; + output: JsonValue; + identity: SchedulerAttemptIdentity; + outgoingEdges: readonly EdgeSpec[]; + }): Promise { + const inputHash = durableJsonHash(fields.input); + const output = snapshotJson(fields.output); + const outputHash = durableJsonHash(output); + const drafts: EventDraft[] = [{ + type: "NodeSucceeded", + nodeId: fields.node.id, + attempt: fields.attempt, + data: { inputHash, output: encodeDurableJson(output), outputHash }, + }]; + for (const edge of [...fields.outgoingEdges].sort((left, right) => + compareUnicodeCodePoints(left.id, right.id), + )) { + drafts.push({ + type: "EdgeEmitted", + nodeId: fields.node.id, + edgeId: edge.id, + attempt: fields.attempt, + data: { outputHash }, + }); + } + await this.append(drafts); + } + + async nodeSettledWithoutAttempt(fields: { + graph: GraphSpec; + node: NodeSpec; + result: NodeRunResult; + }): Promise { + await this.append([{ + type: "NodeSettledWithoutAttempt", + nodeId: fields.node.id, + data: { result: encodeDurableJson(nodeResultDocument(fields.result)) }, + }]); + } + + async runTerminal(result: SchedulerRunResult): Promise { + const type: GraphEventType = result.status === "succeeded" + ? "RunSucceeded" + : result.status === "cancelled" + ? "RunCancelled" + : "RunFailed"; + await this.append([{ + type, + data: { result: encodeDurableJson(resultDocument(result)) }, + }]); + } +} + +const RUNTIME_FAILURE_CODES = new Set([ + "EXECUTOR_NOT_FOUND", + "NODE_EXECUTION_FAILED", + "NODE_TIMEOUT", + "NODE_CANCELLED", + "INVALID_OUTPUT", + "UPSTREAM_FAILED", + "INPUT_BINDING_FAILED", + "ATTEMPT_BUDGET_EXHAUSTED", + "NODE_EXECUTION_INTERRUPTED", +]); + +function booleanValue(value: unknown, runId: string, context: string): boolean { + if (typeof value !== "boolean") { + throw invalidHistory(runId, `${context} must be a boolean`); + } + return value; +} + +function optionalString( + record: Record, + key: string, + runId: string, + context: string, +): string | undefined { + return Object.hasOwn(record, key) + ? stringValue(record[key], runId, `${context}.${key}`) + : undefined; +} + +function decodeFailure(value: unknown, runId: string, context: string): NodeRunFailure { + if (!isRecord(value)) throw invalidHistory(runId, `${context} must be an object`); + const required = ["phase", "code", "message", "nodeId", "attempt", "retryable"]; + const allowed = new Set([...required, "causeName", "upstreamNodeIds"]); + if (required.some((key) => !Object.hasOwn(value, key)) || + Object.keys(value).some((key) => !allowed.has(key))) { + throw invalidHistory(runId, `${context} has invalid failure fields`); + } + if (value.phase !== "execute") { + throw invalidHistory(runId, `${context}.phase must be 'execute'`); + } + const code = stringValue(value.code, runId, `${context}.code`); + if (!RUNTIME_FAILURE_CODES.has(code as NodeRunFailure["code"])) { + throw invalidHistory(runId, `${context}.code is unknown`, { code }); + } + let upstreamNodeIds: string[] | undefined; + if (Object.hasOwn(value, "upstreamNodeIds")) { + upstreamNodeIds = arrayValue(value.upstreamNodeIds, runId, `${context}.upstreamNodeIds`) + .map((item, index) => stringValue(item, runId, `${context}.upstreamNodeIds[${index}]`)); + if (new Set(upstreamNodeIds).size !== upstreamNodeIds.length) { + throw invalidHistory(runId, `${context}.upstreamNodeIds contains duplicates`); + } + } + const causeName = optionalString(value, "causeName", runId, context); + return { + phase: "execute", + code: code as NodeRunFailure["code"], + message: stringValue(value.message, runId, `${context}.message`), + nodeId: stringValue(value.nodeId, runId, `${context}.nodeId`), + attempt: integerValue(value.attempt, runId, `${context}.attempt`), + retryable: booleanValue(value.retryable, runId, `${context}.retryable`), + ...(causeName === undefined ? {} : { causeName }), + ...(upstreamNodeIds === undefined ? {} : { upstreamNodeIds }), + }; +} + +function decodeOutputFailure( + value: unknown, + runId: string, + context: string, +): Extract { + if (!isRecord(value)) throw invalidHistory(runId, `${context} must be an object`); + const required = ["phase", "code", "message", "outputName", "nodeId"]; + const allowed = new Set([...required, "port"]); + if (required.some((key) => !Object.hasOwn(value, key)) || + Object.keys(value).some((key) => !allowed.has(key)) || + value.phase !== "output" || value.code !== "OUTPUT_BINDING_FAILED") { + throw invalidHistory(runId, `${context} has invalid output-failure fields`); + } + const port = optionalString(value, "port", runId, context); + return { + phase: "output", + code: "OUTPUT_BINDING_FAILED", + message: stringValue(value.message, runId, `${context}.message`), + outputName: stringValue(value.outputName, runId, `${context}.outputName`), + nodeId: stringValue(value.nodeId, runId, `${context}.nodeId`), + ...(port === undefined ? {} : { port }), + }; +} + +function decodeNodeResult(value: unknown, runId: string, context: string): NodeRunResult { + if (!isRecord(value)) throw invalidHistory(runId, `${context} must be an object`); + const required = ["nodeId", "sequence", "status", "attempts"]; + const allowed = new Set([...required, "input", "output", "failure"]); + if (required.some((key) => !Object.hasOwn(value, key)) || + Object.keys(value).some((key) => !allowed.has(key))) { + throw invalidHistory(runId, `${context} has invalid node-result fields`); + } + const status = stringValue(value.status, runId, `${context}.status`); + if (status !== "succeeded" && status !== "failed" && status !== "skipped") { + throw invalidHistory(runId, `${context}.status is unknown`); + } + const hasOutput = Object.hasOwn(value, "output"); + const hasFailure = Object.hasOwn(value, "failure"); + if ((status === "succeeded") !== hasOutput || (status === "succeeded") === hasFailure) { + throw invalidHistory(runId, `${context} has an incoherent outcome shape`); + } + return { + nodeId: stringValue(value.nodeId, runId, `${context}.nodeId`), + sequence: integerValue(value.sequence, runId, `${context}.sequence`), + status, + attempts: integerValue(value.attempts, runId, `${context}.attempts`), + ...(Object.hasOwn(value, "input") ? { input: snapshotJson(value.input) } : {}), + ...(hasOutput ? { output: snapshotJson(value.output) } : {}), + ...(hasFailure ? { failure: decodeFailure(value.failure, runId, `${context}.failure`) } : {}), + }; +} + +interface DecodedTerminalResult extends SchedulerRunResult {} + +function decodeTerminalResult( + value: unknown, + runId: string, + compiled: CompiledDurableGraph, +): DecodedTerminalResult { + if (!isRecord(value)) throw invalidHistory(runId, "terminal result must be an object"); + const required = [ + "status", "graphHash", "nodes", "failures", "scheduledOrder", "completionOrder", + "maxObservedConcurrency", "totalAttempts", + ]; + const allowed = new Set([...required, "output"]); + if (required.some((key) => !Object.hasOwn(value, key)) || + Object.keys(value).some((key) => !allowed.has(key))) { + throw invalidHistory(runId, "terminal result has invalid fields"); + } + const status = stringValue(value.status, runId, "terminal result.status"); + if (status !== "succeeded" && status !== "failed" && status !== "cancelled") { + throw invalidHistory(runId, "terminal result status is unknown"); + } + const graphHash = stringValue(value.graphHash, runId, "terminal result.graphHash"); + if (graphHash !== compiled.graphHash) { + throw invalidHistory(runId, "terminal result graph hash is inconsistent"); + } + const nodes = arrayValue(value.nodes, runId, "terminal result.nodes") + .map((item, index) => decodeNodeResult(item, runId, `terminal result.nodes[${index}]`)); + if (nodes.length !== compiled.orderedNodeIds.length || + nodes.some((item, index) => item.nodeId !== compiled.orderedNodeIds[index])) { + throw invalidHistory(runId, "terminal result nodes are not in topological order"); + } + const failures = arrayValue(value.failures, runId, "terminal result.failures") + .map((item, index) => { + if (!isRecord(item)) throw invalidHistory(runId, `terminal result.failures[${index}] must be an object`); + return item.phase === "execute" + ? decodeFailure(item, runId, `terminal result.failures[${index}]`) + : decodeOutputFailure(item, runId, `terminal result.failures[${index}]`); + }); + const stringArray = (field: "scheduledOrder" | "completionOrder"): string[] => + arrayValue(value[field], runId, `terminal result.${field}`) + .map((item, index) => stringValue(item, runId, `terminal result.${field}[${index}]`)); + const output = Object.hasOwn(value, "output") + ? snapshotJson(value.output) as JsonRecord + : undefined; + if (output !== undefined && !isRecord(output)) { + throw invalidHistory(runId, "terminal result.output must be an object"); + } + return { + status, + graphHash, + nodes, + failures, + scheduledOrder: stringArray("scheduledOrder"), + completionOrder: stringArray("completionOrder"), + maxObservedConcurrency: integerValue( + value.maxObservedConcurrency, runId, "terminal result.maxObservedConcurrency", + ), + totalAttempts: integerValue(value.totalAttempts, runId, "terminal result.totalAttempts"), + ...(output === undefined ? {} : { output }), + }; +} + +interface NodeProjection { + readonly nodeId: string; + input: JsonValue | undefined; + inputHash: string | undefined; + activityKey: string | undefined; + sideEffects: ReturnType | undefined; + scheduledAttempt: number | undefined; + openAttempt: number | undefined; + retryAttempt: number | undefined; + retryAvailableAt: string | undefined; + retryReserved: boolean; + attempts: number; + result: NodeRunResult | undefined; + outputHash: string | undefined; +} + +interface FoldedRun { + readonly graphInput: JsonValue; + readonly graphHash: string; + readonly inputHash: string; + readonly implementationHash: string; + readonly maxTotalAttempts: number; + readonly projections: ReadonlyMap; + readonly totalAttempts: number; + readonly scheduledOrder: readonly string[]; + readonly completionOrder: readonly string[]; + readonly maxObservedConcurrency: number; + readonly terminalResult: DecodedTerminalResult | undefined; + readonly eventIds: ReadonlySet; + readonly version: number; +} + +function maxNodeAttempts(node: NodeSpec): number { + return node.retry?.maxAttempts ?? 1; +} + +function sameJson(left: unknown, right: unknown): boolean { + return JSON.stringify(encodeDurableJson(left)) === JSON.stringify(encodeDurableJson(right)); +} + +function sameSettledResultSemantics(left: NodeRunResult, right: NodeRunResult): boolean { + const normalize = (result: NodeRunResult): JsonRecord => { + const document = nodeResultDocument(result); + if (result.failure === undefined) return document; + const failure = failureDocument(result.failure); + delete failure.message; + delete failure.causeName; + return { ...document, failure }; + }; + return sameJson(normalize(left), normalize(right)); +} + +function sameTerminalFailures( + left: readonly GraphRunFailure[], + right: readonly GraphRunFailure[], +): boolean { + if (left.length !== right.length) return false; + return left.every((failure, index) => { + const expected = right[index] as GraphRunFailure; + if (failure.phase !== expected.phase) return false; + if (failure.phase !== "output" || expected.phase !== "output") { + return sameJson(graphFailureDocument(failure), graphFailureDocument(expected)); + } + return failure.code === expected.code && + failure.outputName === expected.outputName && + failure.nodeId === expected.nodeId && + failure.port === expected.port; + }); +} + +function eventNode( + event: GraphEvent, + compiled: CompiledDurableGraph, + runId: string, +): string { + if (event.nodeId === undefined || !compiled.nodesById.has(event.nodeId)) { + throw invalidHistory(runId, `${event.type} references an unknown node`); + } + return event.nodeId; +} + +function eventAttempt(event: GraphEvent, runId: string): number { + if (event.attempt === undefined) { + throw invalidHistory(runId, `${event.type} requires an attempt`); + } + return event.attempt; +} + +function noEventIdentity(event: GraphEvent, runId: string): void { + if (event.nodeId !== undefined || event.edgeId !== undefined || event.attempt !== undefined) { + throw invalidHistory(runId, `${event.type} has unexpected node, edge, or attempt identity`); + } +} + +function declarationOrder( + values: readonly string[], + compiled: CompiledDurableGraph, +): readonly string[] { + return [...values].sort((left, right) => + (compiled.declaration.get(left) ?? Number.MAX_SAFE_INTEGER) - + (compiled.declaration.get(right) ?? Number.MAX_SAFE_INTEGER)); +} + +function runtimeFailure( + nodeId: string, + code: NodeRunFailure["code"], + message: string, + attempt: number, + fields: Partial> = {}, +): NodeRunFailure { + return { + phase: "execute", + nodeId, + code, + message, + attempt, + retryable: fields.retryable ?? false, + ...(fields.causeName === undefined ? {} : { causeName: fields.causeName }), + ...(fields.upstreamNodeIds === undefined ? {} : { upstreamNodeIds: fields.upstreamNodeIds }), + }; +} + +function assertPermutation( + order: readonly string[], + compiled: CompiledDurableGraph, + runId: string, + name: string, +): void { + if (order.length !== compiled.orderedNodeIds.length || + new Set(order).size !== order.length || + order.some((nodeId) => !compiled.nodesById.has(nodeId))) { + throw invalidHistory(runId, `terminal ${name} is not a node permutation`); + } +} + +function reconstructTerminalNodes(fields: { + runId: string; + terminalType: "RunSucceeded" | "RunFailed" | "RunCancelled"; + terminal: DecodedTerminalResult; + compiled: CompiledDurableGraph; + projections: ReadonlyMap; + graphInput: JsonValue; + totalAttempts: number; +}): Map { + const { runId, terminal, compiled, projections } = fields; + const expected = new Map(); + for (const nodeId of compiled.orderedNodeIds) { + const projection = projections.get(nodeId) as NodeProjection; + if (projection.result === undefined) { + throw invalidHistory(runId, "terminal result contains a node with no committed outcome", { + nodeId, + }); + } + expected.set(nodeId, projection.result); + } + for (const terminalNode of terminal.nodes) { + const committed = expected.get(terminalNode.nodeId) as NodeRunResult; + if (!sameJson(nodeResultDocument(terminalNode), nodeResultDocument(committed))) { + throw invalidHistory(runId, "terminal result contradicts folded node history", { + nodeId: terminalNode.nodeId, + }); + } + } + return expected; +} + +function validateTerminalResult(fields: { + runId: string; + terminalType: "RunSucceeded" | "RunFailed" | "RunCancelled"; + terminal: DecodedTerminalResult; + compiled: CompiledDurableGraph; + projections: ReadonlyMap; + graphInput: JsonValue; + scheduledOrder: readonly string[]; + completionOrder: readonly string[]; + maxObservedConcurrency: number; + totalAttempts: number; +}): void { + const { + runId, terminalType, terminal, compiled, projections, graphInput, scheduledOrder, + completionOrder, maxObservedConcurrency, totalAttempts, + } = fields; + const expectedStatus = terminalType === "RunSucceeded" + ? "succeeded" + : terminalType === "RunCancelled" ? "cancelled" : "failed"; + if (terminal.status !== expectedStatus) { + throw invalidHistory(runId, "terminal event and result status disagree"); + } + if (terminal.totalAttempts !== totalAttempts) { + throw invalidHistory(runId, "terminal result attempt count is inconsistent"); + } + if (terminal.maxObservedConcurrency !== maxObservedConcurrency) { + throw invalidHistory(runId, "terminal max concurrency contradicts event history"); + } + if ([...projections.values()].some((item) => item.openAttempt !== undefined)) { + throw invalidHistory(runId, "terminal event leaves an attempt open"); + } + const expectedNodes = reconstructTerminalNodes({ + runId, terminalType, terminal, compiled, projections, graphInput, totalAttempts, + }); + const failures: GraphRunFailure[] = []; + for (const nodeId of compiled.orderedNodeIds) { + const failure = expectedNodes.get(nodeId)?.failure; + if (failure !== undefined) failures.push(failure); + } + const output = Object.create(null) as Record; + let outputsComplete = true; + for (const [name, endpoint] of Object.entries(compiled.graph.outputs) + .sort(([left], [right]) => compareUnicodeCodePoints(left, right))) { + const result = expectedNodes.get(endpoint.node); + if (result?.status !== "succeeded") { + outputsComplete = false; + continue; + } + try { + defineJsonValue(output, name, snapshotJson(endpointValue(endpoint, result.output))); + } catch (error) { + outputsComplete = false; + failures.push({ + phase: "output", + code: "OUTPUT_BINDING_FAILED", + message: `Could not bind graph output '${name}': ${errorMessage(error)}`, + outputName: name, + nodeId: endpoint.node, + ...(endpoint.port === undefined ? {} : { port: endpoint.port }), + }); + } + } + if ((terminal.output === undefined) !== !outputsComplete || + (terminal.output !== undefined && !sameJson(terminal.output, output))) { + throw invalidHistory(runId, "terminal output contradicts committed node outputs"); + } + if (!sameTerminalFailures(terminal.failures, failures)) { + throw invalidHistory(runId, "terminal failures contradict folded node outcomes"); + } + assertPermutation(terminal.scheduledOrder, compiled, runId, "scheduledOrder"); + assertPermutation(terminal.completionOrder, compiled, runId, "completionOrder"); + if (JSON.stringify(terminal.scheduledOrder) !== JSON.stringify(scheduledOrder) || + JSON.stringify(terminal.completionOrder) !== JSON.stringify(completionOrder)) { + throw invalidHistory(runId, "terminal node orders contradict event order"); + } + const succeeded = failures.length === 0 && outputsComplete && + [...expectedNodes.values()].every((item) => item.status === "succeeded"); + if ((terminal.status === "succeeded") !== succeeded && terminal.status !== "cancelled") { + throw invalidHistory(runId, "terminal status contradicts reconstructed graph result"); + } +} + +function validateSettledWithoutAttempt(fields: { + runId: string; + compiled: CompiledDurableGraph; + graphInput: JsonValue; + projections: ReadonlyMap; + projection: NodeProjection; + result: NodeRunResult; + totalAttempts: number; + maxTotalAttempts: number; + pendingRetryReservations: number; +}): void { + const { + runId, compiled, graphInput, projections, projection, result, totalAttempts, + maxTotalAttempts, pendingRetryReservations, + } = fields; + const nodeId = projection.nodeId; + const node = compiled.nodesById.get(nodeId) as NodeSpec; + if (result.nodeId !== nodeId || result.sequence !== compiled.sequence.get(nodeId) || + result.attempts !== projection.attempts || result.status === "succeeded" || + result.failure === undefined || result.failure.nodeId !== nodeId || + result.failure.attempt !== projection.attempts) { + throw invalidHistory(runId, "NodeSettledWithoutAttempt result identity is invalid", { nodeId }); + } + const committed = new Map(); + for (const [id, item] of projections) { + if (item.result !== undefined) committed.set(id, item.result); + } + const incoming = compiled.incoming.get(nodeId) ?? []; + const unresolvedUpstream = [...new Set(incoming + .map((edge) => edge.from.node) + .filter((upstreamId) => !committed.has(upstreamId)))] + .sort(compareUnicodeCodePoints); + if (unresolvedUpstream.length > 0) { + throw invalidHistory( + runId, + "NodeSettledWithoutAttempt was emitted before every dependency settled", + { nodeId, unresolvedUpstreamNodeIds: unresolvedUpstream }, + ); + } + const failedUpstream = [...new Set(incoming + .map((edge) => edge.from.node) + .filter((upstreamId) => committed.get(upstreamId)?.status !== "succeeded"))] + .sort(compareUnicodeCodePoints); + + let expected: NodeRunResult | undefined; + if (result.failure.code === "NODE_CANCELLED") { + if (result.status === "skipped") { + expected = { + nodeId, + sequence: compiled.sequence.get(nodeId) as number, + status: "skipped", + attempts: projection.attempts, + failure: runtimeFailure( + nodeId, + "NODE_CANCELLED", + `Node '${nodeId}' did not start because the run was cancelled`, + projection.attempts, + ), + }; + } else { + let input: JsonValue; + try { + input = bindInput(compiled, nodeId, graphInput, committed); + } catch (error) { + throw invalidHistory(runId, "cancelled node did not have bindable input", { nodeId }, { cause: error }); + } + expected = { + nodeId, + sequence: compiled.sequence.get(nodeId) as number, + status: "failed", + attempts: projection.attempts, + input, + failure: runtimeFailure( + nodeId, "NODE_CANCELLED", `Node '${nodeId}' was cancelled`, projection.attempts, + ), + }; + } + } else if (failedUpstream.length > 0) { + expected = { + nodeId, + sequence: compiled.sequence.get(nodeId) as number, + status: "skipped", + attempts: projection.attempts, + failure: runtimeFailure( + nodeId, + "UPSTREAM_FAILED", + `Node '${nodeId}' did not start because upstream nodes failed: ${failedUpstream.join(", ")}`, + projection.attempts, + { upstreamNodeIds: failedUpstream }, + ), + }; + } else { + let input: JsonValue | undefined; + let bindingError: unknown; + try { + input = bindInput(compiled, nodeId, graphInput, committed); + } catch (error) { + bindingError = error; + } + if (bindingError !== undefined) { + expected = { + nodeId, + sequence: compiled.sequence.get(nodeId) as number, + status: "failed", + attempts: projection.attempts, + failure: runtimeFailure( + nodeId, + "INPUT_BINDING_FAILED", + `Could not bind input for node '${nodeId}': ${errorMessage(bindingError)}`, + projection.attempts, + { causeName: errorName(bindingError) }, + ), + }; + } else if (result.failure.code === "EXECUTOR_NOT_FOUND") { + expected = { + nodeId, + sequence: compiled.sequence.get(nodeId) as number, + status: "failed", + attempts: projection.attempts, + input: input as JsonValue, + failure: runtimeFailure( + nodeId, + "EXECUTOR_NOT_FOUND", + `No executor is registered for node '${nodeId}' (kind '${node.kind}')`, + projection.attempts, + ), + }; + } else if (projection.attempts >= maxNodeAttempts(node)) { + expected = { + nodeId, + sequence: compiled.sequence.get(nodeId) as number, + status: "failed", + attempts: projection.attempts, + input: input as JsonValue, + failure: runtimeFailure( + nodeId, + "ATTEMPT_BUDGET_EXHAUSTED", + `Node '${nodeId}' has exhausted its durable attempt budget`, + projection.attempts, + ), + }; + } else if (totalAttempts + pendingRetryReservations >= maxTotalAttempts) { + expected = { + nodeId, + sequence: compiled.sequence.get(nodeId) as number, + status: "failed", + attempts: projection.attempts, + input: input as JsonValue, + failure: runtimeFailure( + nodeId, + "ATTEMPT_BUDGET_EXHAUSTED", + `Run attempt budget was exhausted before node '${nodeId}' could start`, + projection.attempts, + ), + }; + } + } + if (expected === undefined || !sameSettledResultSemantics(result, expected)) { + throw invalidHistory(runId, "NodeSettledWithoutAttempt result is not scheduler-derived", { + nodeId, + code: result.failure.code, + }); + } +} + +function foldHistory( + events: readonly GraphEvent[], + fields: { + compiled: CompiledDurableGraph; + runId: string; + implementationHash: string; + }, +): FoldedRun { + const { compiled, runId, implementationHash } = fields; + if (events.length === 0) { + throw new DurableRunError("RUN_NOT_FOUND", runId, "durable run does not exist"); + } + const projections = new Map(compiled.graph.nodes.map((node) => [ + node.id, + { + nodeId: node.id, + input: undefined, + inputHash: undefined, + activityKey: undefined, + sideEffects: undefined, + scheduledAttempt: undefined, + openAttempt: undefined, + retryAttempt: undefined, + retryAvailableAt: undefined, + retryReserved: false, + attempts: 0, + result: undefined, + outputHash: undefined, + }, + ])); + let graphInput: JsonValue = null; + let storedGraphHash = ""; + let storedInputHash = ""; + let storedImplementationHash = ""; + let maxTotalAttempts = 0; + let started = false; + let terminalResult: DecodedTerminalResult | undefined; + let terminalSeen = false; + const scheduledOrder: string[] = []; + const completionOrder: string[] = []; + const active = new Set(); + const pendingRetryReservations = new Set(); + let maxObservedConcurrency = 0; + let totalAttempts = 0; + let expectedEdges: Array<{ + edgeId: string; + producerId: string; + attempt: number; + outputHash: string; + }> = []; + let expectedRetry: { nodeId: string; attempt: number; activityKey: string } | undefined; + const eventIds = new Set(); + + for (let index = 0; index < events.length; index += 1) { + const event = events[index] as GraphEvent; + assertGraphEvent(event); + strictDate(event.timestamp, runId, `event[${index}].timestamp`); + if (event.sequence !== index || event.runId !== runId) { + throw invalidHistory(runId, "event identity or sequence is inconsistent", { sequence: index }); + } + if (event.graphRevision !== GRAPH_REVISION) { + throw invalidHistory(runId, "event graph revision is not 1", { sequence: index }); + } + if (eventIds.has(event.eventId)) { + throw invalidHistory(runId, "eventId is duplicated", { eventId: event.eventId }); + } + eventIds.add(event.eventId); + if (event.payloadHash === undefined || event.payloadHash !== canonicalHash(event.data)) { + throw invalidHistory(runId, "event payload hash is invalid", { sequence: index }); + } + if (terminalSeen) { + throw invalidHistory(runId, "event appears after a terminal event", { sequence: index }); + } + + if (expectedEdges.length > 0) { + const expected = expectedEdges.shift() as (typeof expectedEdges)[number]; + if (event.type !== "EdgeEmitted" || event.edgeId !== expected.edgeId || + event.nodeId !== expected.producerId || event.attempt !== expected.attempt) { + throw invalidHistory( + runId, + "NodeSucceeded is not followed by its ordered EdgeEmitted batch", + { sequence: index, expectedEdgeId: expected.edgeId }, + ); + } + exactKeys(event.data, ["outputHash"], runId, "EdgeEmitted.data"); + if (event.data.outputHash !== expected.outputHash) { + throw invalidHistory(runId, "EdgeEmitted output hash is invalid"); + } + continue; + } + + if (expectedRetry !== undefined) { + if (event.type !== "NodeRetried" || event.nodeId !== expectedRetry.nodeId || + event.attempt !== expectedRetry.attempt || event.edgeId !== undefined) { + throw invalidHistory( + runId, + "retryable NodeAttemptFailed is not followed by NodeRetried", + { sequence: index }, + ); + } + exactKeys(event.data, ["availableAt", "activityKey"], runId, "NodeRetried.data"); + const availableAt = stringValue(event.data.availableAt, runId, "NodeRetried.availableAt"); + strictDate(availableAt, runId, "NodeRetried.availableAt"); + if (event.data.activityKey !== expectedRetry.activityKey) { + throw invalidHistory(runId, "NodeRetried activity key is invalid"); + } + const projection = projections.get(expectedRetry.nodeId) as NodeProjection; + const node = compiled.nodesById.get(expectedRetry.nodeId) as NodeSpec; + if (expectedRetry.attempt > maxNodeAttempts(node)) { + throw invalidHistory(runId, "NodeRetried exceeds the node retry budget", { + nodeId: node.id, attempt: expectedRetry.attempt, + }); + } + if (totalAttempts + pendingRetryReservations.size >= maxTotalAttempts) { + throw invalidHistory(runId, "NodeRetried is not eligible under the global attempt budget", { + nodeId: node.id, attempt: expectedRetry.attempt, + }); + } + projection.retryAttempt = expectedRetry.attempt; + projection.retryAvailableAt = availableAt; + projection.retryReserved = true; + pendingRetryReservations.add(expectedRetry.nodeId); + expectedRetry = undefined; + continue; + } + + if (index === 0) { + if (event.type !== "RunCreated") { + throw invalidHistory(runId, "RunCreated must be the first event"); + } + noEventIdentity(event, runId); + exactKeys(event.data, [ + "contractVersion", "graphHash", "implementationHash", "input", "inputHash", + "maxTotalAttempts", + ], runId, "RunCreated.data"); + if (event.data.contractVersion !== CONTRACT_VERSION) { + throw invalidHistory(runId, "RunCreated contract version is incompatible"); + } + storedGraphHash = stringValue(event.data.graphHash, runId, "RunCreated.graphHash"); + storedImplementationHash = stringValue( + event.data.implementationHash, runId, "RunCreated.implementationHash", + ); + storedInputHash = stringValue(event.data.inputHash, runId, "RunCreated.inputHash"); + maxTotalAttempts = integerValue( + event.data.maxTotalAttempts, runId, "RunCreated.maxTotalAttempts", 1, + ); + try { + graphInput = decodeDurableJson(event.data.input); + } catch (error) { + throw new DurableRunError( + "INPUT_HASH_MISMATCH", + runId, + "stored graph input is not valid Durable JSON", + {}, + { cause: error }, + ); + } + if (durableJsonHash(graphInput) !== storedInputHash) { + throw new DurableRunError( + "INPUT_HASH_MISMATCH", runId, "stored graph input does not match inputHash", + ); + } + if (storedGraphHash !== compiled.graphHash) { + throw new DurableRunError( + "GRAPH_HASH_MISMATCH", + runId, + "compiled graph does not match the durable run", + { expected: storedGraphHash, actual: compiled.graphHash }, + ); + } + if (storedImplementationHash !== implementationHash) { + throw new DurableRunError( + "IMPLEMENTATION_MISMATCH", + runId, + "implementationId does not match the durable run", + { expected: storedImplementationHash, actual: implementationHash }, + ); + } + if (maxTotalAttempts !== effectiveAttemptLimit(compiled.graph)) { + throw invalidHistory(runId, "stored attempt budget is inconsistent with graph"); + } + continue; + } + + if (event.type === "RunCreated") { + throw invalidHistory(runId, "RunCreated appears more than once", { sequence: index }); + } + if (event.type === "RunStarted") { + noEventIdentity(event, runId); + if (started || index !== 1 || Object.keys(event.data).length !== 0) { + throw invalidHistory(runId, "RunStarted is duplicate, misplaced, or non-empty"); + } + started = true; + continue; + } + if (!started) { + throw invalidHistory(runId, "lifecycle event appears before RunStarted"); + } + + if (event.type === "RunResumed") { + noEventIdentity(event, runId); + exactKeys(event.data, ["reusedNodeIds", "interruptedNodeIds"], runId, "RunResumed.data"); + const decodeNodeList = (field: "reusedNodeIds" | "interruptedNodeIds"): string[] => { + const values = arrayValue(event.data[field], runId, `RunResumed.${field}`) + .map((item, itemIndex) => stringValue(item, runId, `RunResumed.${field}[${itemIndex}]`)); + if (values.some((nodeId) => !compiled.nodesById.has(nodeId)) || + new Set(values).size !== values.length || + JSON.stringify(values) !== JSON.stringify(declarationOrder(values, compiled))) { + throw invalidHistory(runId, `RunResumed.${field} is invalid or out of declaration order`); + } + return values; + }; + const reused = decodeNodeList("reusedNodeIds"); + const interrupted = decodeNodeList("interruptedNodeIds"); + if (reused.some((nodeId) => interrupted.includes(nodeId))) { + throw invalidHistory(runId, "RunResumed node lists overlap"); + } + const expectedReused = declarationOrder( + [...projections.values()] + .filter((item) => item.result?.status === "succeeded") + .map((item) => item.nodeId), + compiled, + ); + const expectedInterrupted = declarationOrder( + [...projections.values()] + .filter((item) => item.openAttempt !== undefined) + .map((item) => item.nodeId), + compiled, + ); + if (JSON.stringify(reused) !== JSON.stringify(expectedReused)) { + throw invalidHistory(runId, "RunResumed reusedNodeIds contradict history"); + } + if (JSON.stringify(interrupted) !== JSON.stringify(expectedInterrupted)) { + throw invalidHistory(runId, "RunResumed interruptedNodeIds contradict history"); + } + continue; + } + + if (event.type === "NodeSettledWithoutAttempt") { + if (event.edgeId !== undefined || event.attempt !== undefined) { + throw invalidHistory(runId, "NodeSettledWithoutAttempt must omit edgeId and attempt"); + } + const nodeId = eventNode(event, compiled, runId); + const projection = projections.get(nodeId) as NodeProjection; + if (projection.openAttempt !== undefined || projection.result !== undefined) { + throw invalidHistory(runId, "NodeSettledWithoutAttempt targets an open or settled node", { + nodeId, + }); + } + exactKeys(event.data, ["result"], runId, "NodeSettledWithoutAttempt.data"); + let decodedResult: JsonValue; + try { + decodedResult = decodeDurableJson(event.data.result); + } catch (error) { + throw invalidHistory( + runId, "NodeSettledWithoutAttempt result is malformed", {}, { cause: error }, + ); + } + const result = decodeNodeResult( + decodedResult, runId, "NodeSettledWithoutAttempt.result", + ); + pendingRetryReservations.delete(nodeId); + projection.retryReserved = false; + validateSettledWithoutAttempt({ + runId, + compiled, + graphInput, + projections, + projection, + result, + totalAttempts, + maxTotalAttempts, + pendingRetryReservations: pendingRetryReservations.size, + }); + projection.result = result; + projection.scheduledAttempt = undefined; + projection.retryAttempt = undefined; + projection.retryAvailableAt = undefined; + if (!scheduledOrder.includes(nodeId)) scheduledOrder.push(nodeId); + if (!completionOrder.includes(nodeId)) completionOrder.push(nodeId); + continue; + } + + if (event.type === "NodeScheduled") { + if (event.edgeId !== undefined) throw invalidHistory(runId, "NodeScheduled has an edge identity"); + const nodeId = eventNode(event, compiled, runId); + const attempt = eventAttempt(event, runId); + const projection = projections.get(nodeId) as NodeProjection; + if (projection.result !== undefined || projection.openAttempt !== undefined) { + throw invalidHistory(runId, "settled or running node was scheduled again"); + } + if (projection.scheduledAttempt !== undefined && projection.retryAttempt === undefined) { + throw invalidHistory(runId, "NodeScheduled is duplicated without a retry"); + } + const hasRetryReservation = pendingRetryReservations.has(nodeId); + if ((projection.retryAttempt !== undefined) !== hasRetryReservation) { + throw invalidHistory(runId, "NodeScheduled retry reservation is inconsistent", { + nodeId, + }); + } + const expectedAttempt = projection.retryAttempt ?? projection.attempts + 1; + if (attempt !== expectedAttempt) { + throw invalidHistory(runId, "NodeScheduled attempt is non-contiguous"); + } + const node = compiled.nodesById.get(nodeId) as NodeSpec; + if (attempt > maxNodeAttempts(node)) { + throw invalidHistory(runId, "NodeScheduled exceeds the node retry budget", { nodeId, attempt }); + } + if (!hasRetryReservation && + totalAttempts + pendingRetryReservations.size >= maxTotalAttempts) { + throw invalidHistory(runId, "NodeScheduled is not eligible under the global attempt budget", { + nodeId, attempt, + }); + } + exactKeys( + event.data, ["input", "inputHash", "activityKey", "sideEffects"], + runId, "NodeScheduled.data", + ); + let nodeInput: JsonValue; + try { + nodeInput = decodeDurableJson(event.data.input); + } catch (error) { + throw invalidHistory(runId, "NodeScheduled input is malformed", {}, { cause: error }); + } + const inputHash = stringValue(event.data.inputHash, runId, "NodeScheduled.inputHash"); + if (durableJsonHash(nodeInput) !== inputHash) { + throw invalidHistory(runId, "NodeScheduled inputHash is invalid"); + } + let expectedInput: JsonValue; + try { + const committed = new Map(); + for (const [id, item] of projections) { + if (item.result !== undefined) committed.set(id, item.result); + } + expectedInput = bindInput(compiled, nodeId, graphInput, committed); + } catch (error) { + throw invalidHistory( + runId, + "NodeScheduled was emitted before deterministic input binding was ready", + { nodeId, attempt }, + { cause: error }, + ); + } + if (!sameJson(nodeInput, expectedInput)) { + throw invalidHistory(runId, "NodeScheduled input does not match deterministic graph binding", { + nodeId, attempt, + }); + } + if (projection.attempts > 0 && projection.input !== undefined && + !sameJson(nodeInput, projection.input)) { + throw invalidHistory(runId, "retry changed the original bound node input", { nodeId, attempt }); + } + const activityKey = stringValue(event.data.activityKey, runId, "NodeScheduled.activityKey"); + const expectedActivityKey = durableJsonHash([ + "activity/v1alpha1", runId, GRAPH_REVISION, nodeId, inputHash, + ]); + if (activityKey !== expectedActivityKey) { + throw invalidHistory(runId, "NodeScheduled activityKey is invalid"); + } + const declaredSideEffects = stringValue( + event.data.sideEffects, runId, "NodeScheduled.sideEffects", + ); + if (declaredSideEffects !== sideEffects(node)) { + throw invalidHistory(runId, "NodeScheduled sideEffects is inconsistent"); + } + projection.input = nodeInput; + projection.inputHash = inputHash; + projection.activityKey = activityKey; + projection.sideEffects = declaredSideEffects as NodeProjection["sideEffects"]; + projection.scheduledAttempt = attempt; + projection.retryAttempt = undefined; + if (!scheduledOrder.includes(nodeId)) scheduledOrder.push(nodeId); + continue; + } + + if (event.type === "NodeStarted") { + if (event.edgeId !== undefined) throw invalidHistory(runId, "NodeStarted has an edge identity"); + const nodeId = eventNode(event, compiled, runId); + const attempt = eventAttempt(event, runId); + const projection = projections.get(nodeId) as NodeProjection; + if (projection.scheduledAttempt !== attempt || projection.openAttempt !== undefined || + projection.result !== undefined || projection.retryAttempt !== undefined) { + throw invalidHistory(runId, "NodeStarted lacks its required NodeScheduled"); + } + exactKeys(event.data, ["inputHash", "activityKey"], runId, "NodeStarted.data"); + if (event.data.inputHash !== projection.inputHash || + event.data.activityKey !== projection.activityKey) { + throw invalidHistory(runId, "NodeStarted recovery identity is invalid"); + } + if (attempt > maxNodeAttempts(compiled.nodesById.get(nodeId) as NodeSpec)) { + throw invalidHistory(runId, "NodeStarted exceeds the node retry budget", { nodeId, attempt }); + } + const consumedReservation = pendingRetryReservations.delete(nodeId); + if (!consumedReservation && + totalAttempts + pendingRetryReservations.size >= maxTotalAttempts) { + throw invalidHistory(runId, "NodeStarted has no available global attempt slot", { + nodeId, attempt, + }); + } + projection.retryReserved = false; + projection.retryAvailableAt = undefined; + projection.scheduledAttempt = undefined; + projection.openAttempt = attempt; + projection.attempts = attempt; + totalAttempts += 1; + if (totalAttempts + pendingRetryReservations.size > maxTotalAttempts) { + throw invalidHistory(runId, "history exceeds the global attempt budget"); + } + active.add(nodeId); + const concurrencyPolicy = compiled.graph.policies?.maxConcurrency; + if (concurrencyPolicy !== undefined && active.size > concurrencyPolicy) { + throw invalidHistory(runId, "history exceeds the graph concurrency policy", { + active: active.size, + maxConcurrency: concurrencyPolicy, + }); + } + maxObservedConcurrency = Math.max(maxObservedConcurrency, active.size); + continue; + } + + if (event.type === "NodeSucceeded") { + if (event.edgeId !== undefined) throw invalidHistory(runId, "NodeSucceeded has an edge identity"); + const nodeId = eventNode(event, compiled, runId); + const attempt = eventAttempt(event, runId); + const projection = projections.get(nodeId) as NodeProjection; + if (projection.openAttempt !== attempt || projection.result !== undefined) { + throw invalidHistory(runId, "NodeSucceeded lacks one open attempt"); + } + exactKeys(event.data, ["inputHash", "output", "outputHash"], runId, "NodeSucceeded.data"); + if (event.data.inputHash !== projection.inputHash) { + throw invalidHistory(runId, "NodeSucceeded inputHash is invalid"); + } + let output: JsonValue; + try { + output = decodeDurableJson(event.data.output); + } catch (error) { + throw invalidHistory(runId, "NodeSucceeded output is malformed", {}, { cause: error }); + } + const outputHash = stringValue(event.data.outputHash, runId, "NodeSucceeded.outputHash"); + if (durableJsonHash(output) !== outputHash) { + throw invalidHistory(runId, "NodeSucceeded outputHash is invalid"); + } + projection.outputHash = outputHash; + projection.openAttempt = undefined; + active.delete(nodeId); + projection.result = { + nodeId, + sequence: compiled.sequence.get(nodeId) as number, + status: "succeeded", + attempts: attempt, + ...(projection.input === undefined ? {} : { input: projection.input }), + output, + }; + if (!completionOrder.includes(nodeId)) completionOrder.push(nodeId); + expectedEdges = (compiled.outgoing.get(nodeId) ?? []).map((edge) => ({ + edgeId: edge.id, producerId: nodeId, attempt, outputHash, + })); + continue; + } + + if (event.type === "NodeAttemptFailed") { + if (event.edgeId !== undefined) throw invalidHistory(runId, "NodeAttemptFailed has an edge identity"); + const nodeId = eventNode(event, compiled, runId); + const attempt = eventAttempt(event, runId); + const projection = projections.get(nodeId) as NodeProjection; + if (projection.openAttempt !== attempt || projection.result !== undefined) { + throw invalidHistory(runId, "NodeAttemptFailed lacks one open attempt"); + } + exactKeys(event.data, ["terminal", "failure"], runId, "NodeAttemptFailed.data"); + const terminal = booleanValue(event.data.terminal, runId, "NodeAttemptFailed.terminal"); + const failure = decodeFailure(event.data.failure, runId, "NodeAttemptFailed.failure"); + if (failure.nodeId !== nodeId || failure.attempt !== attempt) { + throw invalidHistory(runId, "attempt failure identity is invalid"); + } + const allowedAttemptCodes = new Set([ + "NODE_EXECUTION_FAILED", + "NODE_TIMEOUT", + "NODE_CANCELLED", + "INVALID_OUTPUT", + "NODE_EXECUTION_INTERRUPTED", + ]); + if (!allowedAttemptCodes.has(failure.code)) { + throw invalidHistory(runId, "NodeAttemptFailed uses a non-attempt outcome code", { + nodeId, + code: failure.code, + }); + } + if (pendingRetryReservations.has(nodeId)) { + throw invalidHistory(runId, "open attempt retained a retry reservation", { nodeId }); + } + const retryableCodes = new Set([ + "NODE_EXECUTION_FAILED", "NODE_TIMEOUT", "INVALID_OUTPUT", "NODE_EXECUTION_INTERRUPTED", + ]); + const safeInterrupted = failure.code !== "NODE_EXECUTION_INTERRUPTED" || + projection.sideEffects === "none" || projection.sideEffects === "idempotent"; + const shouldRetry = retryableCodes.has(failure.code) && safeInterrupted && + attempt < maxNodeAttempts(compiled.nodesById.get(nodeId) as NodeSpec) && + totalAttempts + pendingRetryReservations.size < maxTotalAttempts; + if (terminal !== !shouldRetry || failure.retryable !== shouldRetry) { + throw invalidHistory(runId, "attempt retryability contradicts deterministic eligibility", { + nodeId, + attempt, + expectedRetryable: shouldRetry, + }); + } + if (failure.code === "NODE_EXECUTION_INTERRUPTED" && + (failure.message !== "process ended before the attempt outcome was durably recorded" || + failure.causeName !== "ProcessLost")) { + throw invalidHistory(runId, "interrupted attempt failure is not canonical", { nodeId, attempt }); + } + if (failure.code === "NODE_CANCELLED" && (!terminal || failure.retryable)) { + throw invalidHistory(runId, "cancelled attempt failure is not canonical", { nodeId, attempt }); + } + projection.openAttempt = undefined; + active.delete(nodeId); + if (terminal) { + projection.result = { + nodeId, + sequence: compiled.sequence.get(nodeId) as number, + status: "failed", + attempts: attempt, + ...(projection.input === undefined ? {} : { input: projection.input }), + failure, + }; + if (!completionOrder.includes(nodeId)) completionOrder.push(nodeId); + } else { + if (projection.activityKey === undefined) { + throw invalidHistory(runId, "retry has no activity key"); + } + expectedRetry = { nodeId, attempt: attempt + 1, activityKey: projection.activityKey }; + } + continue; + } + + if (event.type === "RunSucceeded" || event.type === "RunFailed" || event.type === "RunCancelled") { + noEventIdentity(event, runId); + if (pendingRetryReservations.size > 0) { + throw invalidHistory(runId, "terminal event leaves retry reservations pending", { + nodeIds: declarationOrder([...pendingRetryReservations], compiled), + }); + } + exactKeys(event.data, ["result"], runId, `${event.type}.data`); + let decoded: JsonValue; + try { + decoded = decodeDurableJson(event.data.result); + } catch (error) { + throw invalidHistory(runId, "terminal result is malformed", {}, { cause: error }); + } + terminalResult = decodeTerminalResult(decoded, runId, compiled); + validateTerminalResult({ + runId, + terminalType: event.type, + terminal: terminalResult, + compiled, + projections, + graphInput, + scheduledOrder, + completionOrder, + maxObservedConcurrency, + totalAttempts, + }); + terminalSeen = true; + continue; + } + + throw invalidHistory( + runId, + `event type '${event.type}' is outside durable DAG recovery`, + { sequence: index }, + ); + } + + if (expectedEdges.length > 0) { + throw invalidHistory(runId, "event stream ends inside an EdgeEmitted batch"); + } + if (expectedRetry !== undefined) { + throw invalidHistory(runId, "event stream ends before required NodeRetried"); + } + if (!started) { + throw invalidHistory(runId, "event stream omits RunStarted"); + } + return { + graphInput, + graphHash: storedGraphHash, + inputHash: storedInputHash, + implementationHash: storedImplementationHash, + maxTotalAttempts, + projections, + totalAttempts, + scheduledOrder, + completionOrder, + maxObservedConcurrency, + terminalResult, + eventIds, + version: (events.at(-1) as GraphEvent).sequence, + }; +} + +async function readHistory(store: EventStore, runId: string): Promise { + try { + const events: GraphEvent[] = []; + for await (const event of store.read(runId)) events.push(event); + return events; + } catch (error) { + if (error instanceof PersistenceError) throw error; + throw new DurableRunError( + "DURABILITY_STORE_FAILED", + runId, + "durable event history could not be read", + { causeName: errorName(error) }, + { cause: error }, + ); + } +} + +function implementationHash(implementationId: string): string { + if (typeof implementationId !== "string" || implementationId.length === 0) { + throw new TypeError("implementationId must be a non-empty string"); + } + return durableJsonHash(implementationId); +} + +function effectiveNow(options: DurableSchedulerOptions): () => Date { + return options.now ?? (() => new Date()); +} + +function effectiveEventIdFactory( + options: DurableSchedulerOptions, +): NonNullable { + return options.createEventId ?? (({ runId, sequence }) => `${runId}:${sequence}`); +} + +function stableOptions(options: DurableSchedulerOptions): DurableSchedulerOptions { + if (typeof options !== "object" || options === null) { + throw new TypeError("durable scheduler options must be an object"); + } + return Object.freeze({ + ...options, + ...(options.nodeExecutors === undefined + ? {} + : { nodeExecutors: Object.freeze({ ...options.nodeExecutors }) }), + ...(options.executors === undefined + ? {} + : { executors: Object.freeze({ ...options.executors }) }), + }); +} + +function schedulerOptions( + options: DurableSchedulerOptions, + extra: Omit, +): SchedulerInternalOptions { + return { + ...(options.nodeExecutors === undefined + ? {} + : { nodeExecutors: options.nodeExecutors as unknown as NonNullable }), + ...(options.executors === undefined + ? {} + : { executors: options.executors as unknown as NonNullable }), + ...(options.concurrency === undefined ? {} : { concurrency: options.concurrency }), + ...(options.signal === undefined ? {} : { signal: options.signal }), + ...extra, + }; +} + +function publicResult(result: SchedulerRunResult): DurableGraphRunResult { + return { + status: result.status, + graphHash: result.graphHash, + ...(result.output === undefined ? {} : { output: result.output }), + nodes: result.nodes, + failures: result.failures, + maxObservedConcurrency: result.maxObservedConcurrency, + totalAttempts: result.totalAttempts, + }; +} + +function remainingDelayMs(availableAt: string, now: Date, runId: string): number { + return Math.max(0, strictDate(availableAt, runId, "retry availability").getTime() - now.getTime()); +} + +/** Create and execute one durable graph run. This operation never resumes an existing stream. */ +export async function startDurableGraphRun( + graph: GraphSpec, + input: unknown, + options: DurableSchedulerOptions, +): Promise { + options = stableOptions(options); + const compiled = compileDurableGraph(graph); + if (isCompilationFailure(compiled)) return compiled; + assertDurableTimerBounds(compiled.graph); + const implementation = implementationHash(options.implementationId); + const inputSnapshot = snapshotJson(input); + const existing = await readHistory(options.eventStore, options.runId); + if (existing.length > 0) { + throw new DurableRunError( + "RUN_ALREADY_EXISTS", options.runId, "durable run already exists", + ); + } + const inputHash = durableJsonHash(inputSnapshot); + const journal = new DurableJournal({ + compiled, + runId: options.runId, + store: options.eventStore, + version: -1, + now: effectiveNow(options), + createEventId: effectiveEventIdFactory(options), + }); + await journal.append([ + { + type: "RunCreated", + data: { + contractVersion: CONTRACT_VERSION, + graphHash: compiled.graphHash, + implementationHash: implementation, + input: encodeDurableJson(inputSnapshot), + inputHash, + maxTotalAttempts: effectiveAttemptLimit(compiled.graph), + }, + }, + { type: "RunStarted", data: {} }, + ], "RUN_ALREADY_EXISTS"); + const result = await runGraphWithJournal( + compiled.graph, + inputSnapshot, + schedulerOptions(options, { journal }), + ); + return publicResult(result); +} + +/** Continue one non-terminal durable graph stream using its originally bound input. */ +export async function resumeDurableGraphRun( + graph: GraphSpec, + options: DurableSchedulerOptions, +): Promise { + options = stableOptions(options); + const compiled = compileDurableGraph(graph); + if (isCompilationFailure(compiled)) return compiled; + assertDurableTimerBounds(compiled.graph); + const implementation = implementationHash(options.implementationId); + const events = await readHistory(options.eventStore, options.runId); + const folded = foldHistory(events, { + compiled, + runId: options.runId, + implementationHash: implementation, + }); + if (folded.terminalResult !== undefined) { + return publicResult(folded.terminalResult); + } + + for (const projection of folded.projections.values()) { + if (projection.result?.failure?.code === "NODE_EXECUTION_INTERRUPTED" && + projection.sideEffects !== "none" && projection.sideEffects !== "idempotent") { + throw new DurableRunError( + "IN_DOUBT_SIDE_EFFECT", + options.runId, + "an interrupted node remains blocked on unsafe side-effect reconciliation", + { + nodeId: projection.nodeId, + attempt: projection.result.attempts, + sideEffects: projection.sideEffects ?? "unspecified", + }, + ); + } + } + + const now = effectiveNow(options); + const journal = new DurableJournal({ + compiled, + runId: options.runId, + store: options.eventStore, + version: folded.version, + now, + createEventId: effectiveEventIdFactory(options), + eventIds: folded.eventIds, + preScheduled: new Map( + [...folded.projections] + .filter(([, projection]) => projection.scheduledAttempt !== undefined && + projection.openAttempt === undefined && projection.retryAttempt === undefined && + projection.result === undefined) + .map(([nodeId, projection]) => [nodeId, projection.scheduledAttempt as number]), + ), + activityKeys: new Map( + [...folded.projections] + .filter(([, projection]) => projection.activityKey !== undefined) + .map(([nodeId, projection]) => [nodeId, projection.activityKey as string]), + ), + }); + const interrupted = declarationOrder( + [...folded.projections.values()] + .filter((projection) => projection.openAttempt !== undefined) + .map((projection) => projection.nodeId), + compiled, + ); + const reused = declarationOrder( + [...folded.projections.values()] + .filter((projection) => projection.result?.status === "succeeded") + .map((projection) => projection.nodeId), + compiled, + ); + await journal.append([{ + type: "RunResumed", + data: { reusedNodeIds: [...reused], interruptedNodeIds: [...interrupted] }, + }]); + + const initialResults = new Map(); + for (const [nodeId, projection] of folded.projections) { + if (projection.result !== undefined) initialResults.set(nodeId, projection.result); + } + const initialCompletionOrder = [...folded.completionOrder]; + const retryDelays = new Map(); + let reservedAttempts = [...folded.projections.values()] + .filter((projection) => projection.retryReserved).length; + + for (const nodeId of interrupted) { + const projection = folded.projections.get(nodeId) as NodeProjection; + const attempt = projection.openAttempt as number; + const node = compiled.nodesById.get(nodeId) as NodeSpec; + const automatic = projection.sideEffects === "none" || projection.sideEffects === "idempotent"; + const canRetry = automatic && attempt < maxNodeAttempts(node) && + folded.totalAttempts + reservedAttempts < folded.maxTotalAttempts; + if (canRetry) reservedAttempts += 1; + const interruptedFailure = runtimeFailure( + nodeId, + "NODE_EXECUTION_INTERRUPTED", + "process ended before the attempt outcome was durably recorded", + attempt, + { retryable: canRetry, causeName: "ProcessLost" }, + ); + const delayMs = canRetry ? retryDelayMs(node, attempt) : 0; + await journal.attemptFailed({ + graph: compiled.graph, + node, + input: projection.input as JsonValue, + failure: interruptedFailure, + identity: { + runId: options.runId, + attemptId: `${options.runId}/${nodeId}/${attempt}`, + activityKey: projection.activityKey as string, + }, + willRetry: canRetry, + retryDelayMs: delayMs, + }); + if (!automatic) { + throw new DurableRunError( + "IN_DOUBT_SIDE_EFFECT", + options.runId, + "an interrupted node may have produced an unsafe external effect", + { + nodeId, + attempt, + sideEffects: projection.sideEffects ?? "unspecified", + }, + ); + } + if (canRetry) { + retryDelays.set( + nodeId, + remainingDelayMs(journal.retryAvailableAt.get(nodeId) as string, now(), options.runId), + ); + } else { + initialResults.set(nodeId, { + nodeId, + sequence: compiled.sequence.get(nodeId) as number, + status: "failed", + attempts: attempt, + ...(projection.input === undefined ? {} : { input: projection.input }), + failure: interruptedFailure, + }); + if (!initialCompletionOrder.includes(nodeId)) initialCompletionOrder.push(nodeId); + } + } + + for (const [nodeId, projection] of folded.projections) { + if (projection.retryReserved && projection.retryAvailableAt !== undefined && + projection.result === undefined) { + retryDelays.set( + nodeId, + remainingDelayMs(projection.retryAvailableAt, now(), options.runId), + ); + } + } + + const result = await runGraphWithJournal( + compiled.graph, + folded.graphInput, + schedulerOptions(options, { + journal, + initialResults, + attemptOffsets: new Map( + [...folded.projections].map(([nodeId, projection]) => [nodeId, projection.attempts]), + ), + initialTotalAttempts: folded.totalAttempts, + initialMaxObservedConcurrency: folded.maxObservedConcurrency, + initialScheduledOrder: folded.scheduledOrder, + initialCompletionOrder, + initialRetryDelaysMs: retryDelays, + }), + ); + return publicResult(result); +} + +export const startGraphRun = startDurableGraphRun; +export const resumeGraphRun = resumeDurableGraphRun; diff --git a/packages/runtime/src/index.ts b/packages/runtime/src/index.ts index 835ef11..865b683 100644 --- a/packages/runtime/src/index.ts +++ b/packages/runtime/src/index.ts @@ -1,4 +1,26 @@ export { runGraph } from "./scheduler.js"; +export { + resumeDurableGraphRun, + resumeGraphRun, + startDurableGraphRun, + startGraphRun, +} from "./durable.js"; +export { + DurableRunError, + type DurableEventIdContext, + type DurableGraphRunResult, + type DurableNodeExecutionContext, + type DurableNodeExecutor, + type DurableRunErrorCode, + type DurableSchedulerOptions, + type SerializedDurableRunError, +} from "./durable-types.js"; +export { + decodeDurableJson, + durableJsonHash, + encodeDurableJson, + type DurableJson, +} from "./durable-json.js"; export type { CompilationRunFailure, GraphRunFailure, diff --git a/packages/runtime/src/scheduler.ts b/packages/runtime/src/scheduler.ts index 62b5beb..c4f98b5 100644 --- a/packages/runtime/src/scheduler.ts +++ b/packages/runtime/src/scheduler.ts @@ -20,6 +20,78 @@ import type { import { snapshotJson } from "./json.js"; const identityExecutor: NodeExecutor = ({ input }) => input; +const MAX_TIMER_DELAY_MS = 2_147_483_647; + +export interface SchedulerAttemptIdentity { + runId: string; + attemptId: string; + activityKey: string; +} + +export interface SchedulerBeforeAttempt { + graph: GraphSpec; + node: NodeSpec; + input: JsonValue; + attempt: number; + signal: AbortSignal; +} + +export interface SchedulerAttemptFailed { + graph: GraphSpec; + node: NodeSpec; + input: JsonValue; + failure: NodeRunFailure; + identity: SchedulerAttemptIdentity; + willRetry: boolean; + retryDelayMs: number; +} + +export interface SchedulerNodeSucceeded { + graph: GraphSpec; + node: NodeSpec; + input: JsonValue; + attempt: number; + output: JsonValue; + identity: SchedulerAttemptIdentity; + outgoingEdges: readonly EdgeSpec[]; +} + +export interface SchedulerNodeSettledWithoutAttempt { + graph: GraphSpec; + node: NodeSpec; + result: NodeRunResult; +} + +export interface SchedulerRunResult extends GraphRunResult { + scheduledOrder: readonly string[]; + completionOrder: readonly string[]; +} + +export interface SchedulerJournal { + beforeAttempt(context: SchedulerBeforeAttempt): Promise; + attemptFailed(context: SchedulerAttemptFailed): Promise; + nodeSucceeded(context: SchedulerNodeSucceeded): Promise; + nodeSettledWithoutAttempt(context: SchedulerNodeSettledWithoutAttempt): Promise; + runTerminal(result: SchedulerRunResult): Promise; +} + +export interface SchedulerInternalOptions extends SchedulerOptions { + journal?: SchedulerJournal; + initialResults?: ReadonlyMap; + attemptOffsets?: ReadonlyMap; + initialTotalAttempts?: number; + initialMaxObservedConcurrency?: number; + initialScheduledOrder?: readonly string[]; + initialCompletionOrder?: readonly string[]; + initialRetryDelaysMs?: ReadonlyMap; +} + +interface JournalNodeExecutionContext extends NodeExecutionContext { + runId?: string; + attemptId?: string; + activityKey?: string; + idempotencyKey?: string; +} function runtimeFailure( nodeId: string, @@ -81,6 +153,26 @@ function resolveConcurrency(graph: GraphSpec, requested: number | undefined): nu return Math.min(desired, policy); } +function assertTimerBounds(graph: GraphSpec): void { + for (const node of graph.nodes) { + for (const [name, value] of [ + ["timeoutMs", node.timeoutMs], + ["retry.initialDelayMs", node.retry?.initialDelayMs], + ["retry.maxDelayMs", node.retry?.maxDelayMs], + ] as const) { + if (value !== undefined && value > MAX_TIMER_DELAY_MS) { + throw new TypeError( + `Node '${node.id}' ${name} exceeds the ${MAX_TIMER_DELAY_MS}ms runtime timer limit`, + ); + } + } + } + const maxDurationMs = graph.policies?.maxDurationMs; + if (typeof maxDurationMs === "number" && maxDurationMs > MAX_TIMER_DELAY_MS) { + throw new TypeError(`maxDurationMs exceeds the ${MAX_TIMER_DELAY_MS}ms runtime timer limit`); + } +} + function endpointValue(endpoint: Endpoint, value: unknown): unknown { if (endpoint.port === undefined) { return value; @@ -91,6 +183,15 @@ function endpointValue(endpoint: Endpoint, value: unknown): unknown { return (value as Record)[endpoint.port]; } +function defineJsonValue(target: Record, key: string, value: JsonValue): void { + Object.defineProperty(target, key, { + value, + enumerable: true, + configurable: true, + writable: true, + }); +} + function bindNodeInput( node: NodeSpec, incoming: readonly EdgeSpec[], @@ -101,18 +202,25 @@ function bindNodeInput( return graphInput; } - const input: Record = {}; + const input = Object.create(null) as Record; for (const edge of incoming) { const key = edge.to.port ?? edge.from.node; if (Object.hasOwn(input, key)) { throw new Error(`Node '${node.id}' receives more than one value for input key '${key}'`); } - input[key] = endpointValue(edge.from, results.get(edge.from.node)?.output) as JsonValue; + defineJsonValue( + input, + key, + endpointValue(edge.from, results.get(edge.from.node)?.output) as JsonValue, + ); } return snapshotJson(input); } function delay(milliseconds: number, signal: AbortSignal | undefined): Promise { + if (signal?.aborted) { + return Promise.reject(signal.reason); + } if (milliseconds <= 0) { return Promise.resolve(); } @@ -142,7 +250,7 @@ type AttemptOutcome = async function executeAttempt( executor: NodeExecutor, - context: NodeExecutionContext, + context: JournalNodeExecutionContext, timeoutMs: number | undefined, ): Promise { if (context.signal.aborted) { @@ -203,25 +311,89 @@ interface ExecuteNodeOptions { executor: NodeExecutor | undefined; signal: AbortSignal; sequence: number; - claimAttempt: () => boolean; + claimAttempt: (nodeId: string) => boolean; + reserveRetry: (nodeId: string) => boolean; + releaseRetryReservation: (nodeId: string) => void; acquireAttemptSlot: (signal: AbortSignal) => Promise<() => void>; + onAttemptStarted: () => void; + onAttemptFinished: () => void; + attemptOffset?: number; + journal?: SchedulerJournal; + journalEnabled?: () => boolean; + outgoingEdges: readonly EdgeSpec[]; + initialRetryDelayMs?: number; } async function executeNode(options: ExecuteNodeOptions): Promise { - const { graph, node, input, executor, signal, sequence, claimAttempt, acquireAttemptSlot } = options; + const { + graph, + node, + input, + executor, + signal, + sequence, + claimAttempt, + reserveRetry, + releaseRetryReservation, + acquireAttemptSlot, + onAttemptStarted, + onAttemptFinished, + attemptOffset = 0, + journal, + journalEnabled = () => true, + outgoingEdges, + initialRetryDelayMs = 0, + } = options; + + const settledWithoutAttempt = async (result: NodeRunResult): Promise => { + if (journal !== undefined && journalEnabled()) { + await journal.nodeSettledWithoutAttempt({ graph, node, result }); + } + return result; + }; + if (executor === undefined) { const failure = runtimeFailure( node.id, "EXECUTOR_NOT_FOUND", `No executor is registered for node '${node.id}' (kind '${node.kind}')`, - 0, + attemptOffset, ); - return { nodeId: node.id, sequence, status: "failed", attempts: 0, input, failure }; + releaseRetryReservation(node.id); + return settledWithoutAttempt({ + nodeId: node.id, + sequence, + status: "failed", + attempts: attemptOffset, + input, + failure, + }); } const maxAttempts = positiveInteger(node.retry?.maxAttempts, 1); let lastFailure: NodeRunFailure | undefined; - let attempts = 0; + let attempts = attemptOffset; + + if (initialRetryDelayMs > 0) { + try { + await delay(initialRetryDelayMs, signal); + } catch { + const failure = runtimeFailure(node.id, "NODE_CANCELLED", `Node '${node.id}' was cancelled`, attempts); + releaseRetryReservation(node.id); + return settledWithoutAttempt({ nodeId: node.id, sequence, status: "failed", attempts, input, failure }); + } + } + + if (attempts >= maxAttempts) { + const failure = runtimeFailure( + node.id, + "ATTEMPT_BUDGET_EXHAUSTED", + `Node '${node.id}' has exhausted its durable attempt budget`, + attempts, + ); + releaseRetryReservation(node.id); + return settledWithoutAttempt({ nodeId: node.id, sequence, status: "failed", attempts, input, failure }); + } while (attempts < maxAttempts) { let releaseAttemptSlot: (() => void) | undefined; @@ -229,14 +401,22 @@ async function executeNode(options: ExecuteNodeOptions): Promise releaseAttemptSlot = await acquireAttemptSlot(signal); } catch { const failure = runtimeFailure(node.id, "NODE_CANCELLED", `Node '${node.id}' was cancelled`, attempts); - return { nodeId: node.id, sequence, status: "failed", attempts, input, failure }; + releaseRetryReservation(node.id); + return settledWithoutAttempt({ nodeId: node.id, sequence, status: "failed", attempts, input, failure }); } - if (!claimAttempt()) { + if (!claimAttempt(node.id)) { releaseAttemptSlot(); if (lastFailure !== undefined) { lastFailure = { ...lastFailure, retryable: false }; - return { nodeId: node.id, sequence, status: "failed", attempts, input, failure: lastFailure }; + return settledWithoutAttempt({ + nodeId: node.id, + sequence, + status: "failed", + attempts, + input, + failure: lastFailure, + }); } const failure = runtimeFailure( node.id, @@ -244,18 +424,83 @@ async function executeNode(options: ExecuteNodeOptions): Promise `Run attempt budget was exhausted before node '${node.id}' could start`, attempts, ); - return { nodeId: node.id, sequence, status: "failed", attempts, input, failure }; + return settledWithoutAttempt({ nodeId: node.id, sequence, status: "failed", attempts, input, failure }); } attempts += 1; let outcome: AttemptOutcome; + let identity: SchedulerAttemptIdentity | undefined; + let attemptMetricStarted = false; + let mayRetry = false; + let retryDelayMs = 0; try { + if (journal !== undefined) { + identity = await journal.beforeAttempt({ graph, node, input, attempt: attempts, signal }); + if (!journalEnabled()) throw signal.reason; + } + onAttemptStarted(); + attemptMetricStarted = true; + const context: JournalNodeExecutionContext = { + graph, + node, + input, + attempt: attempts, + signal, + ...(identity === undefined + ? {} + : { + runId: identity.runId, + attemptId: identity.attemptId, + activityKey: identity.activityKey, + idempotencyKey: identity.activityKey, + }), + }; outcome = await executeAttempt( executor, - { graph, node, input, attempt: attempts, signal }, + context, node.timeoutMs, ); + if (outcome.succeeded && journal !== undefined && identity !== undefined && journalEnabled()) { + await journal.nodeSucceeded({ + graph, + node, + input, + attempt: attempts, + output: outcome.output, + identity, + outgoingEdges, + }); + } else if (!outcome.succeeded) { + const code = outcome.code ?? "NODE_EXECUTION_FAILED"; + mayRetry = attempts < maxAttempts && code !== "NODE_CANCELLED" && reserveRetry(node.id); + lastFailure = runtimeFailure( + node.id, + code, + code === "NODE_TIMEOUT" + ? `Node '${node.id}' timed out after ${node.timeoutMs}ms` + : code === "NODE_CANCELLED" + ? `Node '${node.id}' was cancelled` + : code === "INVALID_OUTPUT" + ? `Node '${node.id}' returned a value that is not portable finite JSON` + : `Node '${node.id}' failed: ${errorMessage(outcome.error)}`, + attempts, + { retryable: mayRetry, causeName: errorName(outcome.error) }, + ); + retryDelayMs = mayRetry ? retryDelay(node, attempts) : 0; + if (journal !== undefined && identity !== undefined && journalEnabled()) { + await journal.attemptFailed({ + graph, + node, + input, + failure: lastFailure, + identity, + willRetry: mayRetry, + retryDelayMs, + }); + } + } } finally { + if (attemptMetricStarted) onAttemptFinished(); releaseAttemptSlot(); } if (outcome.succeeded) { @@ -269,29 +514,30 @@ async function executeNode(options: ExecuteNodeOptions): Promise }; } - const code = outcome.code ?? "NODE_EXECUTION_FAILED"; - const mayRetry = attempts < maxAttempts && code !== "NODE_CANCELLED"; - lastFailure = runtimeFailure( - node.id, - code, - code === "NODE_TIMEOUT" - ? `Node '${node.id}' timed out after ${node.timeoutMs}ms` - : code === "NODE_CANCELLED" - ? `Node '${node.id}' was cancelled` - : code === "INVALID_OUTPUT" - ? `Node '${node.id}' returned a value that is not portable finite JSON` - : `Node '${node.id}' failed: ${errorMessage(outcome.error)}`, - attempts, - { retryable: mayRetry, causeName: errorName(outcome.error) }, - ); + if (!mayRetry) { + return { + nodeId: node.id, + sequence, + status: "failed", + attempts, + input, + failure: lastFailure as NodeRunFailure, + }; + } - if (mayRetry) { - try { - await delay(retryDelay(node, attempts), signal); - } catch { - lastFailure = runtimeFailure(node.id, "NODE_CANCELLED", `Node '${node.id}' was cancelled`, attempts); - break; - } + try { + await delay(retryDelayMs, signal); + } catch { + releaseRetryReservation(node.id); + lastFailure = runtimeFailure(node.id, "NODE_CANCELLED", `Node '${node.id}' was cancelled`, attempts); + return settledWithoutAttempt({ + nodeId: node.id, + sequence, + status: "failed", + attempts, + input, + failure: lastFailure, + }); } } @@ -313,7 +559,7 @@ interface SemaphoreWaiter { onAbort: () => void; } -function createAttemptSemaphore(limit: number, onRunningChange: (running: number) => void) { +function createAttemptSemaphore(limit: number) { let running = 0; const waiters: SemaphoreWaiter[] = []; @@ -323,14 +569,12 @@ function createAttemptSemaphore(limit: number, onRunningChange: (running: number if (released) return; released = true; running -= 1; - onRunningChange(running); while (waiters.length > 0) { const next = waiters.shift() as SemaphoreWaiter; if (next.cancelled) continue; next.signal.removeEventListener("abort", next.onAbort); running += 1; - onRunningChange(running); next.grant(releaseFactory()); break; } @@ -343,7 +587,6 @@ function createAttemptSemaphore(limit: number, onRunningChange: (running: number } if (running < limit) { running += 1; - onRunningChange(running); return Promise.resolve(releaseFactory()); } @@ -364,12 +607,12 @@ function createAttemptSemaphore(limit: number, onRunningChange: (running: number }; } -/** Execute a valid DAG with bounded concurrency and deterministic result ordering. */ -export async function runGraph( +/** Internal scheduler entry point with durable journal and recovery seeds. */ +export async function runGraphWithJournal( graph: GraphSpec, input: unknown, - options: SchedulerOptions = {}, -): Promise { + options: SchedulerInternalOptions = {}, +): Promise { const graphInput = snapshotGraphInput(input); const compilation = compileGraph(graph); if (!compilation.valid) { @@ -383,8 +626,18 @@ export async function runGraph( failures, maxObservedConcurrency: 0, totalAttempts: 0, + scheduledOrder: [], + completionOrder: [], }; } + if (compilation.canonicalGraph === null) { + throw new TypeError("valid graph compilation omitted its canonical graph"); + } + // Bind execution to the exact document whose hash was compiled. This also + // prevents caller or executor mutation from changing live routing/config + // after validation while leaving graphHash unchanged. + graph = snapshotJson(JSON.parse(compilation.canonicalGraph)) as unknown as GraphSpec; + assertTimerBounds(graph); const nodesById = new Map(graph.nodes.map((node) => [node.id, node])); const incoming = new Map(graph.nodes.map((node) => [node.id, [] as EdgeSpec[]])); @@ -408,9 +661,26 @@ export async function runGraph( (sequence.get(left) ?? Number.MAX_SAFE_INTEGER) - (sequence.get(right) ?? Number.MAX_SAFE_INTEGER) || compareUnicodeCodePoints(left, right); - const ready = orderedNodeIds.filter((nodeId) => remaining.get(nodeId) === 0).sort(compareNodes); + const restoredResults = new Map(options.initialResults ?? []); + for (const nodeId of restoredResults.keys()) { + if (!nodesById.has(nodeId)) { + throw new TypeError(`initialResults contains unknown node '${nodeId}'`); + } + } + for (const nodeId of orderedNodeIds) { + if (!restoredResults.has(nodeId)) continue; + for (const edge of outgoing.get(nodeId) ?? []) { + remaining.set(edge.to.node, (remaining.get(edge.to.node) ?? 0) - 1); + } + } + + const ready = orderedNodeIds + .filter((nodeId) => remaining.get(nodeId) === 0 && !restoredResults.has(nodeId)) + .sort(compareNodes); const active = new Map>(); - const results = new Map(); + const results = restoredResults; + const scheduledOrder = [...(options.initialScheduledOrder ?? [])]; + const completionOrder = [...(options.initialCompletionOrder ?? [])]; const concurrency = resolveConcurrency(graph, options.concurrency); const runController = new AbortController(); const cancelRun = () => runController.abort(options.signal?.reason); @@ -420,36 +690,87 @@ export async function runGraph( } const attemptLimit = positiveInteger(graph.policies?.maxTotalAttempts, Number.MAX_SAFE_INTEGER); - let totalAttempts = 0; - let observedConcurrency = 0; - const acquireAttemptSlot = createAttemptSemaphore(concurrency, (running) => { - observedConcurrency = Math.max(observedConcurrency, running); - }); + let totalAttempts = options.initialTotalAttempts ?? 0; + if (!Number.isSafeInteger(totalAttempts) || totalAttempts < 0) { + throw new TypeError("initialTotalAttempts must be a non-negative safe integer"); + } + const retryReservations = new Set(options.initialRetryDelaysMs?.keys() ?? []); + for (const nodeId of retryReservations) { + if (!nodesById.has(nodeId)) { + throw new TypeError(`initialRetryDelaysMs contains unknown node '${nodeId}'`); + } + if (restoredResults.has(nodeId)) { + throw new TypeError(`initial retry reservation targets settled node '${nodeId}'`); + } + } + if (totalAttempts + retryReservations.size > attemptLimit) { + throw new TypeError("initial durable attempts and retry reservations exceed maxTotalAttempts"); + } + let observedConcurrency = options.initialMaxObservedConcurrency ?? 0; + let journalEnabled = true; + let runningAttempts = 0; + const acquireAttemptSlot = createAttemptSemaphore(concurrency); + const onAttemptStarted = (): void => { + runningAttempts += 1; + observedConcurrency = Math.max(observedConcurrency, runningAttempts); + }; + const onAttemptFinished = (): void => { + runningAttempts -= 1; + }; - const claimAttempt = (): boolean => { - if (totalAttempts >= attemptLimit) { + const claimAttempt = (nodeId: string): boolean => { + if (retryReservations.delete(nodeId)) { + totalAttempts += 1; + return true; + } + if (totalAttempts + retryReservations.size >= attemptLimit) { return false; } totalAttempts += 1; return true; }; + const reserveRetry = (nodeId: string): boolean => { + if (retryReservations.has(nodeId)) return false; + if (totalAttempts + retryReservations.size >= attemptLimit) return false; + retryReservations.add(nodeId); + return true; + }; + + const releaseRetryReservation = (nodeId: string): void => { + retryReservations.delete(nodeId); + }; + const settle = (nodeId: string, result: NodeRunResult): void => { results.set(nodeId, result); + completionOrder.push(nodeId); for (const edge of outgoing.get(nodeId) ?? []) { const next = (remaining.get(edge.to.node) ?? 0) - 1; remaining.set(edge.to.node, next); - if (next === 0) { + if (next === 0 && !results.has(edge.to.node)) { ready.push(edge.to.node); } } ready.sort(compareNodes); }; - const launchReady = (): void => { + const settleWithoutAttempt = async ( + node: NodeSpec, + result: NodeRunResult, + ): Promise => { + releaseRetryReservation(node.id); + if (options.journal !== undefined && journalEnabled) { + await options.journal.nodeSettledWithoutAttempt({ graph, node, result }); + } + settle(node.id, result); + }; + + const launchReady = async (): Promise => { while (ready.length > 0) { const nodeId = ready.shift() as string; + if (!scheduledOrder.includes(nodeId)) scheduledOrder.push(nodeId); const node = nodesById.get(nodeId) as NodeSpec; + const attemptOffset = options.attemptOffsets?.get(nodeId) ?? 0; const nodeIncoming = incoming.get(nodeId) ?? []; const failedUpstream = [...new Set( nodeIncoming @@ -465,14 +786,14 @@ export async function runGraph( cancelled ? `Node '${nodeId}' did not start because the run was cancelled` : `Node '${nodeId}' did not start because upstream nodes failed: ${failedUpstream.join(", ")}`, - 0, + attemptOffset, cancelled ? {} : { upstreamNodeIds: failedUpstream }, ); - settle(nodeId, { + await settleWithoutAttempt(node, { nodeId, sequence: sequence.get(nodeId) as number, status: "skipped", - attempts: 0, + attempts: attemptOffset, failure, }); continue; @@ -486,20 +807,28 @@ export async function runGraph( nodeId, "INPUT_BINDING_FAILED", `Could not bind input for node '${nodeId}': ${errorMessage(error)}`, - 0, + attemptOffset, { causeName: errorName(error) }, ); - settle(nodeId, { + await settleWithoutAttempt(node, { nodeId, sequence: sequence.get(nodeId) as number, status: "failed", - attempts: 0, + attempts: attemptOffset, failure, }); continue; } - const executor = options.nodeExecutors?.[nodeId] ?? options.executors?.[node.kind] ?? + const nodeExecutor = options.nodeExecutors !== undefined && + Object.hasOwn(options.nodeExecutors, nodeId) + ? options.nodeExecutors[nodeId] + : undefined; + const kindExecutor = options.executors !== undefined && + Object.hasOwn(options.executors, node.kind) + ? options.executors[node.kind] + : undefined; + const executor = nodeExecutor ?? kindExecutor ?? (node.kind === "transform" || node.kind === "barrier" ? identityExecutor : undefined); const task = executeNode({ graph, @@ -509,7 +838,16 @@ export async function runGraph( signal: runController.signal, sequence: sequence.get(nodeId) as number, claimAttempt, + reserveRetry, + releaseRetryReservation, acquireAttemptSlot, + onAttemptStarted, + onAttemptFinished, + attemptOffset, + ...(options.journal === undefined ? {} : { journal: options.journal }), + journalEnabled: () => journalEnabled, + outgoingEdges: outgoing.get(nodeId) ?? [], + initialRetryDelayMs: options.initialRetryDelaysMs?.get(nodeId) ?? 0, }).then((result) => ({ nodeId, result })); active.set(nodeId, task); } @@ -517,7 +855,7 @@ export async function runGraph( try { while (results.size < graph.nodes.length) { - launchReady(); + await launchReady(); if (active.size === 0) { break; } @@ -525,6 +863,15 @@ export async function runGraph( active.delete(completed.nodeId); settle(completed.nodeId, completed.result); } + } catch (error) { + journalEnabled = false; + runController.abort(error); + // Stop admitting journal writes immediately. Executors are cooperative and + // may ignore AbortSignal, so durability failures must not wait forever for + // unrelated in-flight work. allSettled observes late rejections without + // delaying propagation of the authoritative journal error. + void Promise.allSettled([...active.values()]); + throw error; } finally { options.signal?.removeEventListener("abort", cancelRun); } @@ -533,7 +880,7 @@ export async function runGraph( const failures: GraphRunFailure[] = nodeResults.flatMap((result) => result.failure === undefined ? [] : [result.failure], ); - const output: Record = {}; + const output = Object.create(null) as Record; let outputsComplete = true; for (const [name, endpoint] of Object.entries(graph.outputs).sort(([left], [right]) => compareUnicodeCodePoints(left, right), @@ -544,7 +891,7 @@ export async function runGraph( continue; } try { - output[name] = endpointValue(endpoint, result.output) as JsonValue; + defineJsonValue(output, name, endpointValue(endpoint, result.output) as JsonValue); } catch (error) { outputsComplete = false; failures.push({ @@ -560,13 +907,41 @@ export async function runGraph( const cancelled = runController.signal.aborted; const succeeded = !cancelled && failures.length === 0 && outputsComplete; - return { + const runResult: SchedulerRunResult = { status: cancelled ? "cancelled" : succeeded ? "succeeded" : "failed", graphHash: compilation.graphHash, - ...(outputsComplete ? { output } : {}), + ...(outputsComplete + ? { output: snapshotJson(output) as Readonly> } + : {}), nodes: nodeResults, failures, maxObservedConcurrency: observedConcurrency, totalAttempts, + scheduledOrder, + completionOrder, + }; + if (options.journal !== undefined) { + await options.journal.runTerminal(runResult); + } + return runResult; +} + +/** Execute a valid DAG with bounded concurrency and deterministic result ordering. */ +export async function runGraph( + graph: GraphSpec, + input: unknown, + options: SchedulerOptions = {}, +): Promise { + // Whitelist the public surface at runtime. JavaScript callers must not be + // able to smuggle internal recovery seeds or a journal through an untyped + // options object and bypass durable-history validation. + const publicOptions: SchedulerOptions = { + ...(options.nodeExecutors === undefined ? {} : { nodeExecutors: options.nodeExecutors }), + ...(options.executors === undefined ? {} : { executors: options.executors }), + ...(options.concurrency === undefined ? {} : { concurrency: options.concurrency }), + ...(options.signal === undefined ? {} : { signal: options.signal }), }; + const { scheduledOrder: _scheduledOrder, completionOrder: _completionOrder, ...result } = + await runGraphWithJournal(graph, input, publicOptions); + return result; } diff --git a/packages/runtime/src/types.ts b/packages/runtime/src/types.ts index c9168a7..8e2cfa3 100644 --- a/packages/runtime/src/types.ts +++ b/packages/runtime/src/types.ts @@ -34,7 +34,8 @@ export type RuntimeFailureCode = | "INVALID_OUTPUT" | "UPSTREAM_FAILED" | "INPUT_BINDING_FAILED" - | "ATTEMPT_BUDGET_EXHAUSTED"; + | "ATTEMPT_BUDGET_EXHAUSTED" + | "NODE_EXECUTION_INTERRUPTED"; export interface CompilationRunFailure { phase: "compile"; diff --git a/packages/runtime/test/durable-json.test.ts b/packages/runtime/test/durable-json.test.ts new file mode 100644 index 0000000..606e1bb --- /dev/null +++ b/packages/runtime/test/durable-json.test.ts @@ -0,0 +1,113 @@ +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; +import { + decodeDurableJson, + durableJsonHash, + encodeDurableJson, +} from "../src/durable-json.js"; + +interface DurableJsonCorpus { + validCases: Array<{ + name: string; + source: { kind: "json"; value: unknown } | { kind: "float64Bits"; bits: string }; + expect: { encoding: unknown; canonicalJson: string; sha256: string }; + }>; + invalidEncodedCases: Array<{ name: string; encoding: unknown }>; +} + +function corpus(): DurableJsonCorpus { + const path = fileURLToPath( + new URL("../../../spec/conformance/durable-json.case.json", import.meta.url), + ); + return JSON.parse(readFileSync(path, "utf8")) as DurableJsonCorpus; +} + +function floatFromBits(bits: string): number { + const bytes = new Uint8Array(8); + for (let index = 0; index < bytes.length; index += 1) { + bytes[index] = Number.parseInt(bits.slice(index * 2, index * 2 + 2), 16); + } + return new DataView(bytes.buffer).getFloat64(0, false); +} + +describe("Tagged Durable JSON", () => { + it("round-trips every portable value and preserves binary64 bits", () => { + const value = { + z: [null, true, "text", 42, 1.5, -0.125], + a: { nested: 0 }, + }; + const encoded = encodeDurableJson(value); + expect(encoded).toEqual([ + "o", + [ + ["a", ["o", [["nested", ["i", 0]]]]], + [ + "z", + [ + "a", + [["n"], ["b", true], ["s", "text"], ["i", 42], ["f", "3ff8000000000000"], ["f", "bfc0000000000000"]], + ], + ], + ], + ]); + expect(decodeDurableJson(encoded)).toEqual(value); + expect(Object.isFrozen(encoded)).toBe(true); + }); + + it("normalizes negative zero and hashes the tagged bytes deterministically", () => { + expect(encodeDurableJson(-0)).toEqual(["i", 0]); + expect(durableJsonHash({ b: 2, a: 1 })).toBe(durableJsonHash({ a: 1, b: 2 })); + expect(durableJsonHash(1.5)).toMatch(/^[a-f0-9]{64}$/); + }); + + it.each([ + [["x"]], + [["n", null]], + [["i", 1.5]], + [["f", "3FF8000000000000"]], + [["f", "3ff0000000000000"]], + [["a", "not-an-array"]], + [["o", [["b", ["i", 2]], ["a", ["i", 1]]]]], + [["o", [["a", ["i", 1]], ["a", ["i", 2]]]]], + ])("rejects malformed or non-canonical tagged data %#", (value) => { + expect(() => decodeDurableJson(value)).toThrow(TypeError); + }); + + it("keeps the existing portable JSON rejection boundary", () => { + expect(() => encodeDurableJson(Number.NaN)).toThrow(TypeError); + expect(() => encodeDurableJson(Number.MAX_SAFE_INTEGER + 1)).toThrow(TypeError); + expect(() => encodeDurableJson(1n)).toThrow(TypeError); + }); + + it("passes every valid vector in the shared cross-language corpus", () => { + for (const testCase of corpus().validCases) { + const value = testCase.source.kind === "json" + ? testCase.source.value + : floatFromBits(testCase.source.bits); + const encoded = encodeDurableJson(value); + expect(encoded, testCase.name).toEqual(testCase.expect.encoding); + expect(JSON.stringify(encoded), testCase.name).toBe(testCase.expect.canonicalJson); + expect(durableJsonHash(value), testCase.name).toBe(testCase.expect.sha256); + const decoded = decodeDurableJson(encoded); + if (Object.is(value, -0)) expect(Object.is(decoded, 0), testCase.name).toBe(true); + else expect(decoded, testCase.name).toEqual(value); + } + }); + + it("rejects every invalid encoding in the shared cross-language corpus", () => { + for (const testCase of corpus().invalidEncodedCases) { + expect(() => decodeDurableJson(testCase.encoding), testCase.name).toThrow(TypeError); + } + }); + + it("round-trips an own __proto__ key without prototype mutation", () => { + const source = JSON.parse('{"__proto__":{"polluted":true},"safe":1}') as Record; + const decoded = decodeDurableJson(encodeDurableJson(source)) as Record; + expect(Object.getPrototypeOf(decoded)).toBe(Object.prototype); + expect(Object.hasOwn(decoded, "__proto__")).toBe(true); + expect(decoded.__proto__).toEqual({ polluted: true }); + expect(({} as { polluted?: boolean }).polluted).toBeUndefined(); + expect(Object.isFrozen(decoded)).toBe(true); + }); +}); diff --git a/packages/runtime/test/durable.test.ts b/packages/runtime/test/durable.test.ts new file mode 100644 index 0000000..5c7aceb --- /dev/null +++ b/packages/runtime/test/durable.test.ts @@ -0,0 +1,1342 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { canonicalHash, type GraphSpec, type NodeSpec } from "@graph-engineering/core"; +import { + MemoryEventStore, + VersionConflictError, + type EventStore, + type GraphEvent, +} from "@graph-engineering/persistence"; +import { describe, expect, it, vi } from "vitest"; +import { + decodeDurableJson, + DurableRunError, + durableJsonHash, + encodeDurableJson, + resumeDurableGraphRun, + startDurableGraphRun, + type DurableNodeExecutionContext, +} from "../src/index.js"; + +const FIXED_TIME = "2026-07-26T12:00:00.000Z"; +const fixedNow = (): Date => new Date(FIXED_TIME); + +function node(id: string, overrides: Partial = {}): NodeSpec { + return { + id, + kind: "agent", + inputSchema: {}, + outputSchema: {}, + config: {}, + sideEffects: "none", + ...overrides, + }; +} + +function graph(overrides: Partial = {}): GraphSpec { + return { + apiVersion: "graphengineering.reacher-z.github.io/v1alpha1", + kind: "Graph", + metadata: { name: "durable-test", version: "1" }, + inputSchema: {}, + outputSchema: {}, + entrypoints: ["root"], + outputs: { result: { node: "root" } }, + nodes: [node("root")], + edges: [], + ...overrides, + }; +} + +async function history(store: EventStore, runId: string): Promise { + const result: GraphEvent[] = []; + for await (const event of store.read(runId)) result.push(event); + return result; +} + +function resign(event: GraphEvent, data: Readonly>): GraphEvent { + return { ...event, data, payloadHash: canonicalHash(data) }; +} + +function forgedEvent(fields: { + runId: string; + sequence: number; + type: GraphEvent["type"]; + data: Readonly>; + nodeId?: string; + edgeId?: string; + attempt?: number; +}): GraphEvent { + return { + apiVersion: "graphengineering.reacher-z.github.io/events/v1alpha1", + eventId: `forged:${fields.sequence}`, + type: fields.type, + timestamp: FIXED_TIME, + runId: fields.runId, + graphRevision: 1, + sequence: fields.sequence, + payloadHash: canonicalHash(fields.data), + redacted: true, + data: fields.data, + ...(fields.nodeId === undefined ? {} : { nodeId: fields.nodeId }), + ...(fields.edgeId === undefined ? {} : { edgeId: fields.edgeId }), + ...(fields.attempt === undefined ? {} : { attempt: fields.attempt }), + }; +} + +async function copyHistory(runId: string, events: readonly GraphEvent[]): Promise { + const store = new MemoryEventStore(); + await store.append(runId, -1, events); + return store; +} + +class CommitThenThrowStore implements EventStore { + readonly delegate: MemoryEventStore; + readonly shouldThrow: (events: readonly GraphEvent[]) => boolean; + threw = false; + + constructor( + delegate: MemoryEventStore, + shouldThrow: (events: readonly GraphEvent[]) => boolean, + ) { + this.delegate = delegate; + this.shouldThrow = shouldThrow; + } + + async append( + runId: string, + expectedVersion: number, + events: readonly GraphEvent[], + ): Promise { + const version = await this.delegate.append(runId, expectedVersion, events); + if (!this.threw && this.shouldThrow(events)) { + this.threw = true; + throw new Error("simulated process loss after durable commit"); + } + return version; + } + + read(runId: string, fromSequence = 0): AsyncIterable { + return this.delegate.read(runId, fromSequence); + } +} + +class ConflictStore implements EventStore { + readonly delegate: MemoryEventStore; + + constructor(delegate: MemoryEventStore) { + this.delegate = delegate; + } + + async append(runId: string, expectedVersion: number): Promise { + const actual = (await history(this.delegate, runId)).length - 1; + throw new VersionConflictError(runId, expectedVersion, actual + 1); + } + + read(runId: string, fromSequence = 0): AsyncIterable { + return this.delegate.read(runId, fromSequence); + } +} + +class BlockingSuccessStore implements EventStore { + readonly delegate = new MemoryEventStore(); + readonly entered: Promise; + #enter!: () => void; + #release!: () => void; + #gate: Promise; + #blocked = false; + + constructor() { + this.entered = new Promise((resolve) => { this.#enter = resolve; }); + this.#gate = new Promise((resolve) => { this.#release = resolve; }); + } + + release(): void { + this.#release(); + } + + async append( + runId: string, + expectedVersion: number, + events: readonly GraphEvent[], + ): Promise { + if (!this.#blocked && events.some((event) => event.type === "NodeSucceeded")) { + this.#blocked = true; + this.#enter(); + await this.#gate; + } + return this.delegate.append(runId, expectedVersion, events); + } + + read(runId: string, fromSequence = 0): AsyncIterable { + return this.delegate.read(runId, fromSequence); + } +} + +describe("durable graph scheduler", () => { + it("commits every claim before execution and terminal resume is deeply identical", async () => { + const store = new MemoryEventStore(); + let identity: readonly string[] | undefined; + const result = await startDurableGraphRun(graph(), { seed: 1.5 }, { + runId: "start-basic", + implementationId: "handlers@1", + eventStore: store, + now: fixedNow, + nodeExecutors: { + root: async (context) => { + const events = await history(store, "start-basic"); + expect(events.at(-1)?.type).toBe("NodeStarted"); + identity = [context.runId, context.attemptId, context.idempotencyKey]; + return { decimal: 0.1 }; + }, + }, + }); + + const events = await history(store, "start-basic"); + expect(events.map((event) => event.type)).toEqual([ + "RunCreated", "RunStarted", "NodeScheduled", "NodeStarted", "NodeSucceeded", "RunSucceeded", + ]); + expect(identity).toEqual([ + "start-basic", + "start-basic/root/1", + events[2]?.data.activityKey, + ]); + const mustNotRun = vi.fn(() => { throw new Error("terminal resume executed work"); }); + const resumed = await resumeDurableGraphRun(graph(), { + runId: "start-basic", + implementationId: "handlers@1", + eventStore: store, + now: fixedNow, + nodeExecutors: { root: mustNotRun }, + }); + assert.deepStrictEqual(resumed, result); + expect(mustNotRun).not.toHaveBeenCalled(); + expect((await history(store, "start-basic")).length).toBe(events.length); + }); + + it("passes the shared resume fixture and reuses committed work", async () => { + const fixturePath = fileURLToPath( + new URL("../../../spec/conformance/durable-resume.case.json", import.meta.url), + ); + const fixture = JSON.parse(readFileSync(fixturePath, "utf8")) as { + runId: string; + implementationId: string; + graph: GraphSpec; + graphInput: unknown; + resumeReturns: Record; + expect: { + executorCalls: Array<{ nodeId: string; attempt: number }>; + mergeInput: unknown; + output: unknown; + totalAttempts: number; + }; + }; + const delegate = new MemoryEventStore(); + const crashStore = new CommitThenThrowStore( + delegate, + (batch) => batch.some((event) => event.type === "NodeStarted" && event.nodeId === "right"), + ); + await expect(startDurableGraphRun(fixture.graph, fixture.graphInput, { + runId: fixture.runId, + implementationId: fixture.implementationId, + eventStore: crashStore, + concurrency: 1, + now: fixedNow, + nodeExecutors: { + left: () => ({ left: true }), + right: () => { throw new Error("right executor must not run before simulated loss"); }, + merge: () => ({ merged: false }), + }, + })).rejects.toMatchObject({ code: "DURABILITY_STORE_FAILED" }); + + const preResume = await history(delegate, fixture.runId); + expect(preResume.map((event) => event.type)).toEqual([ + "RunCreated", "RunStarted", "NodeScheduled", "NodeStarted", "NodeSucceeded", "EdgeEmitted", + "NodeScheduled", "NodeStarted", + ]); + const calls: Array<{ nodeId: string; attempt: number }> = []; + let mergeInput: unknown; + const left = vi.fn(() => { throw new Error("committed node executed again"); }); + const resumed = await resumeDurableGraphRun(fixture.graph, { + runId: fixture.runId, + implementationId: fixture.implementationId, + eventStore: crashStore, + now: fixedNow, + nodeExecutors: { + left, + right: (context) => { + calls.push({ nodeId: context.node.id, attempt: context.attempt }); + return fixture.resumeReturns.right; + }, + merge: (context) => { + calls.push({ nodeId: context.node.id, attempt: context.attempt }); + mergeInput = context.input; + return fixture.resumeReturns.merge; + }, + }, + }); + expect(left).not.toHaveBeenCalled(); + expect(calls).toEqual(fixture.expect.executorCalls); + expect(mergeInput).toEqual(fixture.expect.mergeInput); + expect(resumed.output).toEqual(fixture.expect.output); + expect(resumed.totalAttempts).toBe(fixture.expect.totalAttempts); + const after = await history(delegate, fixture.runId); + expect(after.slice(preResume.length, preResume.length + 3).map((event) => event.type)).toEqual([ + "RunResumed", "NodeAttemptFailed", "NodeRetried", + ]); + expect((after.find((event) => event.type === "RunResumed")?.data)).toEqual({ + reusedNodeIds: ["left"], interruptedNodeIds: ["right"], + }); + const terminal = await resumeDurableGraphRun(fixture.graph, { + runId: fixture.runId, + implementationId: fixture.implementationId, + eventStore: crashStore, + nodeExecutors: { + left: left, + right: left, + merge: left, + }, + }); + assert.deepStrictEqual(terminal, resumed); + expect((await history(delegate, fixture.runId)).length).toBe(after.length); + }); + + it("commits success before a dependent is released", async () => { + const store = new BlockingSuccessStore(); + const dependent = vi.fn(() => "done"); + const chain = graph({ + entrypoints: ["root"], + outputs: { result: { node: "dependent" } }, + nodes: [node("root"), node("dependent")], + edges: [{ id: "root-dependent", from: { node: "root" }, to: { node: "dependent" } }], + }); + const running = startDurableGraphRun(chain, { value: 1 }, { + runId: "commit-before-release", + implementationId: "v1", + eventStore: store, + nodeExecutors: { root: () => "root", dependent }, + }); + await store.entered; + expect(dependent).not.toHaveBeenCalled(); + expect((await history(store, "commit-before-release")).some( + (event) => event.type === "NodeSucceeded", + )).toBe(false); + store.release(); + await expect(running).resolves.toMatchObject({ status: "succeeded" }); + expect(dependent).toHaveBeenCalledOnce(); + }); + + it("fails closed for an interrupted undeclared side effect and remains sticky", async () => { + const unsafe = graph({ + nodes: [{ + id: "root", kind: "agent", inputSchema: {}, outputSchema: {}, config: {}, + retry: { maxAttempts: 2 }, + }], + }); + const delegate = new MemoryEventStore(); + const crashStore = new CommitThenThrowStore( + delegate, + (batch) => batch.some((event) => event.type === "NodeStarted"), + ); + await expect(startDurableGraphRun(unsafe, {}, { + runId: "unsafe", + implementationId: "v1", + eventStore: crashStore, + nodeExecutors: { root: () => "never" }, + })).rejects.toMatchObject({ code: "DURABILITY_STORE_FAILED" }); + const executor = vi.fn(() => "must-not-run"); + await expect(resumeDurableGraphRun(unsafe, { + runId: "unsafe", + implementationId: "v1", + eventStore: crashStore, + nodeExecutors: { root: executor }, + })).rejects.toMatchObject({ code: "IN_DOUBT_SIDE_EFFECT" }); + expect(executor).not.toHaveBeenCalled(); + const count = (await history(delegate, "unsafe")).length; + await expect(resumeDurableGraphRun(unsafe, { + runId: "unsafe", + implementationId: "v1", + eventStore: crashStore, + nodeExecutors: { root: executor }, + })).rejects.toMatchObject({ code: "IN_DOUBT_SIDE_EFFECT" }); + expect((await history(delegate, "unsafe")).length).toBe(count); + expect(executor).not.toHaveBeenCalled(); + }); + + it("losing the resume CAS invokes no executor", async () => { + const retrying = graph({ nodes: [node("root", { retry: { maxAttempts: 2 } })] }); + const delegate = new MemoryEventStore(); + const crashStore = new CommitThenThrowStore( + delegate, + (batch) => batch.some((event) => event.type === "NodeStarted"), + ); + await expect(startDurableGraphRun(retrying, {}, { + runId: "cas", + implementationId: "v1", + eventStore: crashStore, + nodeExecutors: { root: () => "never" }, + })).rejects.toBeInstanceOf(DurableRunError); + const executor = vi.fn(() => "bad"); + await expect(resumeDurableGraphRun(retrying, { + runId: "cas", + implementationId: "v1", + eventStore: new ConflictStore(delegate), + nodeExecutors: { root: executor }, + })).rejects.toMatchObject({ code: "RESUME_CONFLICT" }); + expect(executor).not.toHaveBeenCalled(); + }); + + it("validates zero-attempt outcomes independently of current executor maps", async () => { + const store = new MemoryEventStore(); + const missing = graph(); + const failed = await startDurableGraphRun(missing, {}, { + runId: "missing-handler", + implementationId: "missing@1", + eventStore: store, + }); + expect(failed.failures[0]).toMatchObject({ code: "EXECUTOR_NOT_FOUND" }); + expect((await history(store, "missing-handler")).map((event) => event.type)).toEqual([ + "RunCreated", "RunStarted", "NodeSettledWithoutAttempt", "RunFailed", + ]); + const withHandler = await resumeDurableGraphRun(missing, { + runId: "missing-handler", + implementationId: "missing@1", + eventStore: store, + nodeExecutors: { root: () => "would-change-the-outcome" }, + }); + const withoutHandler = await resumeDurableGraphRun(missing, { + runId: "missing-handler", + implementationId: "missing@1", + eventStore: store, + }); + assert.deepStrictEqual(withHandler, failed); + assert.deepStrictEqual(withoutHandler, failed); + }); + + it("snapshots the executor registry and gives handlers an immutable canonical graph", async () => { + const store = new MemoryEventStore(); + const original = vi.fn((context: DurableNodeExecutionContext) => { + expect(Object.isFrozen(context.graph)).toBe(true); + expect(Object.isFrozen(context.graph.nodes)).toBe(true); + expect(Object.isFrozen(context.graph.nodes[0])).toBe(true); + expect(() => { + (context.graph.nodes as NodeSpec[])[0]!.id = "mutated"; + }).toThrow(TypeError); + return "original"; + }); + const replacement = vi.fn(() => "replacement"); + const registry: Record = { root: original }; + const running = startDurableGraphRun(graph(), {}, { + runId: "immutable-inputs", + implementationId: "v1", + eventStore: store, + nodeExecutors: registry, + }); + registry.root = replacement; + const result = await running; + expect(result.output).toEqual({ result: "original" }); + expect(original).toHaveBeenCalledOnce(); + expect(replacement).not.toHaveBeenCalled(); + }); + + it("rejects implementation, graph, input, and payload identity changes", async () => { + const source = new MemoryEventStore(); + await startDurableGraphRun(graph(), { seed: 1 }, { + runId: "identity", + implementationId: "v1", + eventStore: source, + now: fixedNow, + nodeExecutors: { root: () => "ok" }, + }); + await expect(resumeDurableGraphRun(graph(), { + runId: "identity", implementationId: "v2", eventStore: source, + })).rejects.toMatchObject({ code: "IMPLEMENTATION_MISMATCH" }); + await expect(resumeDurableGraphRun(graph({ + nodes: [node("root", { config: { changed: true } })], + }), { + runId: "identity", implementationId: "v1", eventStore: source, + })).rejects.toMatchObject({ code: "GRAPH_HASH_MISMATCH" }); + + const events = await history(source, "identity"); + const createdData = { ...events[0]!.data, inputHash: "0".repeat(64) }; + const changedInput = await copyHistory("identity", [ + resign(events[0]!, createdData), ...events.slice(1), + ]); + await expect(resumeDurableGraphRun(graph(), { + runId: "identity", implementationId: "v1", eventStore: changedInput, + })).rejects.toMatchObject({ code: "INPUT_HASH_MISMATCH" }); + + const badPayload = await copyHistory("identity", [ + ...events.slice(0, 2), + { ...events[2]!, payloadHash: "f".repeat(64) }, + ...events.slice(3), + ]); + await expect(resumeDurableGraphRun(graph(), { + runId: "identity", implementationId: "v1", eventStore: badPayload, + })).rejects.toMatchObject({ code: "INVALID_RUN_HISTORY" }); + }); + + it("rejects resigned terminal and self-consistent scheduled-input forgeries", async () => { + const source = new MemoryEventStore(); + await startDurableGraphRun(graph(), { seed: 1 }, { + runId: "forgery", + implementationId: "v1", + eventStore: source, + now: fixedNow, + nodeExecutors: { root: () => "real" }, + }); + const events = await history(source, "forgery"); + const terminalIndex = events.length - 1; + const terminalResult = JSON.parse(JSON.stringify( + decodeDurableJson(events[terminalIndex]!.data.result), + )) as Record; + terminalResult.output = { result: "forged" }; + const forgedTerminal = await copyHistory("forgery", [ + ...events.slice(0, terminalIndex), + resign(events[terminalIndex]!, { result: encodeDurableJson(terminalResult) }), + ]); + await expect(resumeDurableGraphRun(graph(), { + runId: "forgery", implementationId: "v1", eventStore: forgedTerminal, + nodeExecutors: { root: () => "real" }, + })).rejects.toMatchObject({ code: "INVALID_RUN_HISTORY" }); + + const scheduledIndex = events.findIndex((event) => event.type === "NodeScheduled"); + const startedIndex = events.findIndex((event) => event.type === "NodeStarted"); + const succeededIndex = events.findIndex((event) => event.type === "NodeSucceeded"); + const forgedInput = { seed: 999 }; + const inputHash = durableJsonHash(forgedInput); + const activityKey = durableJsonHash([ + "activity/v1alpha1", "forgery", 1, "root", inputHash, + ]); + const changed = [...events]; + changed[scheduledIndex] = resign(events[scheduledIndex]!, { + input: encodeDurableJson(forgedInput), inputHash, activityKey, sideEffects: "none", + }); + changed[startedIndex] = resign(events[startedIndex]!, { inputHash, activityKey }); + changed[succeededIndex] = resign(events[succeededIndex]!, { + ...events[succeededIndex]!.data, inputHash, + }); + const forgedInputStore = await copyHistory("forgery", changed); + await expect(resumeDurableGraphRun(graph(), { + runId: "forgery", implementationId: "v1", eventStore: forgedInputStore, + })).rejects.toMatchObject({ code: "INVALID_RUN_HISTORY" }); + }); + + it("accounts for a pending retry when another node reaches the global budget", async () => { + const limited = graph({ + entrypoints: ["a", "b"], + outputs: { a: { node: "a" }, b: { node: "b" } }, + nodes: [node("a", { retry: { maxAttempts: 2 } }), node("b")], + policies: { maxConcurrency: 1, maxTotalAttempts: 2 }, + }); + let aAttempts = 0; + const b = vi.fn(() => "must-not-run"); + const store = new MemoryEventStore(); + const result = await startDurableGraphRun(limited, {}, { + runId: "reserved-budget", + implementationId: "v1", + eventStore: store, + concurrency: 1, + now: fixedNow, + nodeExecutors: { + a: () => { + aAttempts += 1; + if (aAttempts === 1) throw new Error("retry me"); + return "a-ok"; + }, + b, + }, + }); + expect(result.status).toBe("failed"); + expect(result.totalAttempts).toBe(2); + expect(aAttempts).toBe(2); + expect(b).not.toHaveBeenCalled(); + expect((await history(store, "reserved-budget")).some( + (event) => event.type === "NodeSettledWithoutAttempt" && event.nodeId === "b", + )).toBe(true); + const terminal = await resumeDurableGraphRun(limited, { + runId: "reserved-budget", implementationId: "v1", eventStore: store, + nodeExecutors: { a: () => "unused", b: () => "unused" }, + }); + assert.deepStrictEqual(terminal, result); + }); + + it("reserves the last retry slot across multiple interrupted nodes and validates retry flags", async () => { + const twoRoots = graph({ + entrypoints: ["a", "b"], + outputs: { a: { node: "a" }, b: { node: "b" } }, + nodes: [ + node("a", { retry: { maxAttempts: 2 } }), + node("b", { retry: { maxAttempts: 2 } }), + ], + policies: { maxConcurrency: 2, maxTotalAttempts: 3 }, + }); + const delegate = new MemoryEventStore(); + const crashStore = new CommitThenThrowStore( + delegate, + (batch) => batch.some((event) => event.type === "NodeStarted" && event.nodeId === "b"), + ); + const waitForAbort = ({ signal }: DurableNodeExecutionContext): Promise => + new Promise((_, reject) => { + signal.addEventListener("abort", () => reject(signal.reason), { once: true }); + }); + await expect(startDurableGraphRun(twoRoots, {}, { + runId: "last-token", + implementationId: "v1", + eventStore: crashStore, + concurrency: 2, + now: fixedNow, + nodeExecutors: { a: waitForAbort, b: () => "never" }, + })).rejects.toMatchObject({ code: "DURABILITY_STORE_FAILED" }); + const before = await history(delegate, "last-token"); + expect(before.filter((event) => event.type === "NodeStarted")).toHaveLength(2); + + const calls: Array<[string, number]> = []; + const result = await resumeDurableGraphRun(twoRoots, { + runId: "last-token", + implementationId: "v1", + eventStore: crashStore, + now: fixedNow, + nodeExecutors: { + a: (context) => { + calls.push([context.node.id, context.attempt]); + return "a-ok"; + }, + b: (context) => { + calls.push([context.node.id, context.attempt]); + return "b-must-not-run"; + }, + }, + }); + expect(calls).toEqual([["a", 2]]); + expect(result.totalAttempts).toBe(3); + expect(result.nodes.find((item) => item.nodeId === "b")?.failure).toMatchObject({ + code: "NODE_EXECUTION_INTERRUPTED", retryable: false, + }); + const events = await history(delegate, "last-token"); + const interrupted = events.filter((event) => + event.type === "NodeAttemptFailed" && + (event.data.failure as { code?: string }).code === "NODE_EXECUTION_INTERRUPTED"); + expect(interrupted.map((event) => [ + event.nodeId, + event.data.terminal, + (event.data.failure as { retryable: boolean }).retryable, + ])).toEqual([ + ["a", false, true], + ["b", true, false], + ]); + + const aIndex = events.indexOf(interrupted[0]!); + const forgedEarlyTerminal = [...events.slice(0, aIndex + 1)]; + const aFailure = interrupted[0]!.data.failure as Record; + forgedEarlyTerminal[aIndex] = resign(interrupted[0]!, { + terminal: true, + failure: { ...aFailure, retryable: false }, + }); + const earlyStore = await copyHistory("last-token", forgedEarlyTerminal); + await expect(resumeDurableGraphRun(twoRoots, { + runId: "last-token", implementationId: "v1", eventStore: earlyStore, + })).rejects.toMatchObject({ code: "INVALID_RUN_HISTORY" }); + + const bIndex = events.indexOf(interrupted[1]!); + const forgedExtraRetry = [...events.slice(0, bIndex + 1)]; + const bFailure = interrupted[1]!.data.failure as Record; + forgedExtraRetry[bIndex] = resign(interrupted[1]!, { + terminal: false, + failure: { ...bFailure, retryable: true }, + }); + const extraStore = await copyHistory("last-token", forgedExtraRetry); + await expect(resumeDurableGraphRun(twoRoots, { + runId: "last-token", implementationId: "v1", eventStore: extraStore, + })).rejects.toMatchObject({ code: "INVALID_RUN_HISTORY" }); + + const retriedIndex = events.findIndex((event) => + event.type === "NodeRetried" && event.nodeId === "a"); + const invalidDate = [...events]; + invalidDate[retriedIndex] = resign(events[retriedIndex]!, { + ...events[retriedIndex]!.data, + availableAt: "2026-02-31T12:00:00.000Z", + }); + const invalidDateStore = await copyHistory("last-token", invalidDate); + await expect(resumeDurableGraphRun(twoRoots, { + runId: "last-token", implementationId: "v1", eventStore: invalidDateStore, + })).rejects.toMatchObject({ code: "INVALID_RUN_HISTORY" }); + + const yearZero = [...events]; + yearZero[retriedIndex] = resign(events[retriedIndex]!, { + ...events[retriedIndex]!.data, + availableAt: "0000-01-01T00:00:00.000Z", + }); + const yearZeroStore = await copyHistory("last-token", yearZero); + await expect(resumeDurableGraphRun(twoRoots, { + runId: "last-token", implementationId: "v1", eventStore: yearZeroStore, + })).rejects.toMatchObject({ code: "INVALID_RUN_HISTORY" }); + }); + + it("rejects non-attempt codes in NodeAttemptFailed", async () => { + const retrying = graph({ nodes: [node("root", { retry: { maxAttempts: 2 } })] }); + const delegate = new MemoryEventStore(); + const crashStore = new CommitThenThrowStore( + delegate, + (batch) => batch.some((event) => event.type === "NodeStarted"), + ); + await expect(startDurableGraphRun(retrying, {}, { + runId: "bad-attempt-code", + implementationId: "v1", + eventStore: crashStore, + nodeExecutors: { root: () => "never" }, + })).rejects.toBeInstanceOf(DurableRunError); + const events = await history(delegate, "bad-attempt-code"); + const suffix = forgedEvent({ + runId: "bad-attempt-code", + sequence: events.length, + type: "NodeAttemptFailed", + nodeId: "root", + attempt: 1, + data: { + terminal: true, + failure: { + phase: "execute", + code: "EXECUTOR_NOT_FOUND", + message: "forged", + nodeId: "root", + attempt: 1, + retryable: false, + }, + }, + }); + await delegate.append("bad-attempt-code", events.length - 1, [suffix]); + await expect(resumeDurableGraphRun(retrying, { + runId: "bad-attempt-code", implementationId: "v1", eventStore: delegate, + })).rejects.toMatchObject({ code: "INVALID_RUN_HISTORY" }); + }); + + it("rejects a dependant settlement before every upstream outcome exists", async () => { + const chain = graph({ + outputs: { result: { node: "dependent" } }, + nodes: [node("root", { retry: { maxAttempts: 2 } }), node("dependent")], + edges: [{ id: "root-dependent", from: { node: "root" }, to: { node: "dependent" } }], + }); + const delegate = new MemoryEventStore(); + const crashStore = new CommitThenThrowStore( + delegate, + (batch) => batch.some((event) => event.type === "NodeStarted" && event.nodeId === "root"), + ); + await expect(startDurableGraphRun(chain, {}, { + runId: "premature-dependent", + implementationId: "v1", + eventStore: crashStore, + nodeExecutors: { root: () => "never", dependent: () => "never" }, + })).rejects.toBeInstanceOf(DurableRunError); + const events = await history(delegate, "premature-dependent"); + const result = { + nodeId: "dependent", + sequence: 1, + status: "skipped", + attempts: 0, + failure: { + phase: "execute", + code: "UPSTREAM_FAILED", + message: "Node 'dependent' did not start because upstream nodes failed: root", + nodeId: "dependent", + attempt: 0, + retryable: false, + upstreamNodeIds: ["root"], + }, + }; + await delegate.append("premature-dependent", events.length - 1, [forgedEvent({ + runId: "premature-dependent", + sequence: events.length, + type: "NodeSettledWithoutAttempt", + nodeId: "dependent", + data: { result: encodeDurableJson(result) }, + })]); + await expect(resumeDurableGraphRun(chain, { + runId: "premature-dependent", implementationId: "v1", eventStore: delegate, + })).rejects.toMatchObject({ code: "INVALID_RUN_HISTORY" }); + }); + + it("does not resolve inherited executor properties for durable nodes", async () => { + const inheritedName = graph({ + entrypoints: ["toString"], + outputs: { result: { node: "toString" } }, + nodes: [node("toString")], + }); + const result = await startDurableGraphRun(inheritedName, {}, { + runId: "own-executors", + implementationId: "v1", + eventStore: new MemoryEventStore(), + nodeExecutors: {}, + executors: {}, + }); + expect(result.status).toBe("failed"); + expect(result.failures[0]).toMatchObject({ code: "EXECUTOR_NOT_FOUND", nodeId: "toString" }); + }); + + it("preserves a persisted retry reservation after NodeScheduled and avoids duplicate scheduling", async () => { + const retrying = graph({ + nodes: [node("root", { retry: { maxAttempts: 2 } })], + policies: { maxTotalAttempts: 2 }, + }); + const delegate = new MemoryEventStore(); + const crashStore = new CommitThenThrowStore( + delegate, + (batch) => batch.some((event) => event.type === "NodeRetried"), + ); + await expect(startDurableGraphRun(retrying, { seed: 1 }, { + runId: "pre-scheduled-retry", + implementationId: "v1", + eventStore: crashStore, + now: fixedNow, + nodeExecutors: { root: () => { throw new Error("retry"); } }, + })).rejects.toMatchObject({ code: "DURABILITY_STORE_FAILED" }); + const events = await history(delegate, "pre-scheduled-retry"); + const firstSchedule = events.find((event) => event.type === "NodeScheduled")!; + await delegate.append("pre-scheduled-retry", events.length - 1, [forgedEvent({ + runId: "pre-scheduled-retry", + sequence: events.length, + type: "NodeScheduled", + nodeId: "root", + attempt: 2, + data: firstSchedule.data, + })]); + const calls: number[] = []; + const result = await resumeDurableGraphRun(retrying, { + runId: "pre-scheduled-retry", + implementationId: "v1", + eventStore: crashStore, + now: fixedNow, + nodeExecutors: { + root: (context) => { + calls.push(context.attempt); + return "ok"; + }, + }, + }); + expect(calls).toEqual([2]); + expect(result.totalAttempts).toBe(2); + const complete = await history(delegate, "pre-scheduled-retry"); + expect(complete.filter((event) => + event.type === "NodeScheduled" && event.nodeId === "root")).toHaveLength(2); + expect(complete.filter((event) => + event.type === "NodeStarted" && event.nodeId === "root")).toHaveLength(2); + }); + + it("records a pre-cancelled pending retry with its historical attempt offset", async () => { + const retrying = graph({ + nodes: [node("root", { retry: { maxAttempts: 2 } })], + policies: { maxTotalAttempts: 2 }, + }); + const delegate = new MemoryEventStore(); + const crashStore = new CommitThenThrowStore( + delegate, + (batch) => batch.some((event) => event.type === "NodeRetried"), + ); + await expect(startDurableGraphRun(retrying, {}, { + runId: "cancel-pending-retry", + implementationId: "v1", + eventStore: crashStore, + now: fixedNow, + nodeExecutors: { root: () => { throw new Error("retry"); } }, + })).rejects.toBeInstanceOf(DurableRunError); + const controller = new AbortController(); + controller.abort(new Error("cancel before resume")); + const executor = vi.fn(() => "must-not-run"); + const result = await resumeDurableGraphRun(retrying, { + runId: "cancel-pending-retry", + implementationId: "v1", + eventStore: crashStore, + signal: controller.signal, + now: fixedNow, + nodeExecutors: { root: executor }, + }); + expect(executor).not.toHaveBeenCalled(); + expect(result.status).toBe("cancelled"); + expect(result.nodes[0]).toMatchObject({ + nodeId: "root", + status: "skipped", + attempts: 1, + failure: { code: "NODE_CANCELLED", attempt: 1 }, + }); + const events = await history(delegate, "cancel-pending-retry"); + const settled = events.find((event) => event.type === "NodeSettledWithoutAttempt")!; + expect(decodeDurableJson(settled.data.result)).toMatchObject({ + nodeId: "root", attempts: 1, failure: { attempt: 1 }, + }); + const terminal = await resumeDurableGraphRun(retrying, { + runId: "cancel-pending-retry", + implementationId: "v1", + eventStore: crashStore, + }); + assert.deepStrictEqual(terminal, result); + }); + + it("accepts foreign human text while keeping NodeSettled semantics strict", async () => { + const source = new MemoryEventStore(); + await startDurableGraphRun(graph(), {}, { + runId: "foreign-settled-text", + implementationId: "v1", + eventStore: source, + now: fixedNow, + }); + const events = await history(source, "foreign-settled-text"); + const settledIndex = events.findIndex((event) => event.type === "NodeSettledWithoutAttempt"); + const terminalIndex = events.length - 1; + const settledResult = JSON.parse(JSON.stringify( + decodeDurableJson(events[settledIndex]!.data.result), + )) as { failure: Record }; + settledResult.failure.message = "python-style: executor missing"; + settledResult.failure.causeName = "ForeignRuntime"; + const terminalResult = JSON.parse(JSON.stringify( + decodeDurableJson(events[terminalIndex]!.data.result), + )) as { + nodes: Array<{ failure: Record }>; + failures: Array>; + }; + terminalResult.nodes[0]!.failure = { ...settledResult.failure }; + terminalResult.failures[0] = { ...settledResult.failure }; + const foreign = [...events]; + foreign[settledIndex] = resign(events[settledIndex]!, { + result: encodeDurableJson(settledResult), + }); + foreign[terminalIndex] = resign(events[terminalIndex]!, { + result: encodeDurableJson(terminalResult), + }); + const store = await copyHistory("foreign-settled-text", foreign); + await expect(resumeDurableGraphRun(graph(), { + runId: "foreign-settled-text", implementationId: "v1", eventStore: store, + })).resolves.toMatchObject({ status: "failed" }); + }); + + it("accepts a foreign cancelled-attempt message but preserves cancellation structure", async () => { + const source = new MemoryEventStore(); + const controller = new AbortController(); + await startDurableGraphRun(graph(), {}, { + runId: "foreign-cancel-text", + implementationId: "v1", + eventStore: source, + now: fixedNow, + signal: controller.signal, + nodeExecutors: { + root: () => { + controller.abort(new Error("stop")); + return "ignored"; + }, + }, + }); + const events = await history(source, "foreign-cancel-text"); + const failedIndex = events.findIndex((event) => event.type === "NodeAttemptFailed"); + const terminalIndex = events.length - 1; + const failure = events[failedIndex]!.data.failure as Record; + const foreignFailure = { ...failure, message: "node 'root' was cancelled" }; + const terminalResult = JSON.parse(JSON.stringify( + decodeDurableJson(events[terminalIndex]!.data.result), + )) as { + nodes: Array<{ failure: Record }>; + failures: Array>; + }; + terminalResult.nodes[0]!.failure = { ...foreignFailure }; + terminalResult.failures[0] = { ...foreignFailure }; + const foreign = [...events]; + foreign[failedIndex] = resign(events[failedIndex]!, { + terminal: true, failure: foreignFailure, + }); + foreign[terminalIndex] = resign(events[terminalIndex]!, { + result: encodeDurableJson(terminalResult), + }); + const store = await copyHistory("foreign-cancel-text", foreign); + await expect(resumeDurableGraphRun(graph(), { + runId: "foreign-cancel-text", implementationId: "v1", eventStore: store, + nodeExecutors: { root: () => "unused" }, + })).resolves.toMatchObject({ status: "cancelled" }); + }); + + it("compares output-binding failures without runtime-specific human text", async () => { + const outputGraph = graph({ outputs: { result: { node: "root", port: "missing" } } }); + const source = new MemoryEventStore(); + await startDurableGraphRun(outputGraph, {}, { + runId: "foreign-output-text", + implementationId: "v1", + eventStore: source, + now: fixedNow, + nodeExecutors: { root: () => ({ actual: true }) }, + }); + const events = await history(source, "foreign-output-text"); + const terminalIndex = events.length - 1; + const terminalResult = JSON.parse(JSON.stringify( + decodeDurableJson(events[terminalIndex]!.data.result), + )) as { failures: Array> }; + terminalResult.failures[0]!.message = "foreign runtime output binding wording"; + const foreign = [...events]; + foreign[terminalIndex] = resign(events[terminalIndex]!, { + result: encodeDurableJson(terminalResult), + }); + const store = await copyHistory("foreign-output-text", foreign); + await expect(resumeDurableGraphRun(outputGraph, { + runId: "foreign-output-text", implementationId: "v1", eventStore: store, + nodeExecutors: { root: () => "unused" }, + })).resolves.toMatchObject({ + status: "failed", + failures: [expect.objectContaining({ code: "OUTPUT_BINDING_FAILED" })], + }); + }); + + it("durably transports __proto__ as an own edge and graph-output key", async () => { + const outputs = JSON.parse('{"__proto__":{"node":"consumer"}}') as GraphSpec["outputs"]; + const document = graph({ + outputs, + nodes: [node("root"), node("consumer")], + edges: [{ + id: "root-consumer", + from: { node: "root" }, + to: { node: "consumer", port: "__proto__" }, + }], + }); + const store = new MemoryEventStore(); + let boundInput: Record | undefined; + const result = await startDurableGraphRun(document, {}, { + runId: "durable-proto-key", + implementationId: "v1", + eventStore: store, + nodeExecutors: { + root: () => ({ safe: true }), + consumer: ({ input }) => { + boundInput = input as Record; + return { received: Object.hasOwn(input as object, "__proto__") }; + }, + }, + }); + expect(result.status).toBe("succeeded"); + expect(Object.hasOwn(boundInput as object, "__proto__")).toBe(true); + expect(boundInput?.__proto__).toEqual({ safe: true }); + expect(Object.getPrototypeOf(boundInput)).toBe(Object.prototype); + expect(Object.hasOwn(result.output as object, "__proto__")).toBe(true); + expect(result.output?.__proto__).toEqual({ received: true }); + const terminal = await resumeDurableGraphRun(document, { + runId: "durable-proto-key", + implementationId: "v1", + eventStore: store, + }); + assert.deepStrictEqual(terminal, result); + }); + + it("rejects duplicate or invalid writer event IDs before committing their batch", async () => { + for (const [runId, factory] of [ + ["duplicate-initial-ids", () => "duplicate"], + ["invalid-initial-id", () => ""], + ["throwing-id-factory", () => { throw new Error("factory failed"); }], + ] as const) { + const store = new MemoryEventStore(); + const executor = vi.fn(() => "never"); + await expect(startDurableGraphRun(graph(), {}, { + runId, + implementationId: "v1", + eventStore: store, + createEventId: factory, + nodeExecutors: { root: executor }, + })).rejects.toMatchObject({ code: "DURABILITY_STORE_FAILED" }); + expect(await history(store, runId)).toEqual([]); + expect(executor).not.toHaveBeenCalled(); + } + }); + + it("maps throwing and invalid writer clocks to a latched durability failure", async () => { + for (const [runId, now] of [ + ["throwing-clock", () => { throw new Error("clock failed"); }], + ["invalid-clock", () => new Date(Number.NaN)], + ] as const) { + const store = new MemoryEventStore(); + const executor = vi.fn(() => "never"); + await expect(startDurableGraphRun(graph(), {}, { + runId, + implementationId: "v1", + eventStore: store, + now, + nodeExecutors: { root: executor }, + })).rejects.toMatchObject({ code: "DURABILITY_STORE_FAILED" }); + expect(await history(store, runId)).toEqual([]); + expect(executor).not.toHaveBeenCalled(); + } + }); + + it("rejects a resume event ID that collides with committed history", async () => { + const retrying = graph({ nodes: [node("root", { retry: { maxAttempts: 2 } })] }); + const delegate = new MemoryEventStore(); + const crashStore = new CommitThenThrowStore( + delegate, + (batch) => batch.some((event) => event.type === "NodeStarted"), + ); + await expect(startDurableGraphRun(retrying, {}, { + runId: "resume-id-collision", + implementationId: "v1", + eventStore: crashStore, + nodeExecutors: { root: () => "never" }, + })).rejects.toBeInstanceOf(DurableRunError); + const before = await history(delegate, "resume-id-collision"); + const executor = vi.fn(() => "never"); + await expect(resumeDurableGraphRun(retrying, { + runId: "resume-id-collision", + implementationId: "v1", + eventStore: crashStore, + createEventId: () => before[0]!.eventId, + nodeExecutors: { root: executor }, + })).rejects.toMatchObject({ code: "DURABILITY_STORE_FAILED" }); + expect(await history(delegate, "resume-id-collision")).toEqual(before); + expect(executor).not.toHaveBeenCalled(); + }); + + it("rejects a second NodeStarted after success, terminal failure, or NodeRetried", async () => { + const successSource = new MemoryEventStore(); + await startDurableGraphRun(graph(), {}, { + runId: "duplicate-start-success", + implementationId: "v1", + eventStore: successSource, + nodeExecutors: { root: () => "ok" }, + }); + const successEvents = await history(successSource, "duplicate-start-success"); + const successIndex = successEvents.findIndex((event) => event.type === "NodeSucceeded"); + const originalStart = successEvents.find((event) => event.type === "NodeStarted")!; + const successPrefix = successEvents.slice(0, successIndex + 1); + const forgedSuccess = await copyHistory("duplicate-start-success", [ + ...successPrefix, + forgedEvent({ + runId: "duplicate-start-success", + sequence: successPrefix.length, + type: "NodeStarted", + nodeId: "root", + attempt: 1, + data: originalStart.data, + }), + ]); + await expect(resumeDurableGraphRun(graph(), { + runId: "duplicate-start-success", implementationId: "v1", eventStore: forgedSuccess, + })).rejects.toMatchObject({ code: "INVALID_RUN_HISTORY" }); + + const failureSource = new MemoryEventStore(); + await startDurableGraphRun(graph(), {}, { + runId: "duplicate-start-failure", + implementationId: "v1", + eventStore: failureSource, + nodeExecutors: { root: () => { throw new Error("failed"); } }, + }); + const failureEvents = await history(failureSource, "duplicate-start-failure"); + const failedIndex = failureEvents.findIndex((event) => event.type === "NodeAttemptFailed"); + const failurePrefix = failureEvents.slice(0, failedIndex + 1); + const failedStart = failureEvents.find((event) => event.type === "NodeStarted")!; + const forgedFailure = await copyHistory("duplicate-start-failure", [ + ...failurePrefix, + forgedEvent({ + runId: "duplicate-start-failure", + sequence: failurePrefix.length, + type: "NodeStarted", + nodeId: "root", + attempt: 1, + data: failedStart.data, + }), + ]); + await expect(resumeDurableGraphRun(graph(), { + runId: "duplicate-start-failure", implementationId: "v1", eventStore: forgedFailure, + })).rejects.toMatchObject({ code: "INVALID_RUN_HISTORY" }); + + const retrying = graph({ nodes: [node("root", { retry: { maxAttempts: 2 } })] }); + const retryDelegate = new MemoryEventStore(); + const retryCrash = new CommitThenThrowStore( + retryDelegate, + (batch) => batch.some((event) => event.type === "NodeRetried"), + ); + await expect(startDurableGraphRun(retrying, {}, { + runId: "duplicate-start-retry", + implementationId: "v1", + eventStore: retryCrash, + nodeExecutors: { root: () => { throw new Error("retry"); } }, + })).rejects.toBeInstanceOf(DurableRunError); + const retryEvents = await history(retryDelegate, "duplicate-start-retry"); + const retryStart = retryEvents.find((event) => event.type === "NodeStarted")!; + await retryDelegate.append("duplicate-start-retry", retryEvents.length - 1, [forgedEvent({ + runId: "duplicate-start-retry", + sequence: retryEvents.length, + type: "NodeStarted", + nodeId: "root", + attempt: 1, + data: retryStart.data, + })]); + await expect(resumeDurableGraphRun(retrying, { + runId: "duplicate-start-retry", implementationId: "v1", eventStore: retryDelegate, + })).rejects.toMatchObject({ code: "INVALID_RUN_HISTORY" }); + }); + + it("rejects history whose open attempts exceed the graph concurrency policy", async () => { + const limited = graph({ + entrypoints: ["a", "b"], + outputs: { a: { node: "a" }, b: { node: "b" } }, + nodes: [node("a", { retry: { maxAttempts: 2 } }), node("b")], + policies: { maxConcurrency: 1, maxTotalAttempts: 2 }, + }); + const delegate = new MemoryEventStore(); + const crashStore = new CommitThenThrowStore( + delegate, + (batch) => batch.some((event) => event.type === "NodeStarted" && event.nodeId === "a"), + ); + await expect(startDurableGraphRun(limited, {}, { + runId: "forged-concurrency", + implementationId: "v1", + eventStore: crashStore, + concurrency: 1, + nodeExecutors: { a: () => "never", b: () => "never" }, + })).rejects.toBeInstanceOf(DurableRunError); + const events = await history(delegate, "forged-concurrency"); + const inputHash = durableJsonHash({}); + const activityKey = durableJsonHash([ + "activity/v1alpha1", "forged-concurrency", 1, "b", inputHash, + ]); + await delegate.append("forged-concurrency", events.length - 1, [ + forgedEvent({ + runId: "forged-concurrency", + sequence: events.length, + type: "NodeScheduled", + nodeId: "b", + attempt: 1, + data: { + input: encodeDurableJson({}), inputHash, activityKey, sideEffects: "none", + }, + }), + forgedEvent({ + runId: "forged-concurrency", + sequence: events.length + 1, + type: "NodeStarted", + nodeId: "b", + attempt: 1, + data: { inputHash, activityKey }, + }), + ]); + await expect(resumeDurableGraphRun(limited, { + runId: "forged-concurrency", implementationId: "v1", eventStore: delegate, + })).rejects.toMatchObject({ code: "INVALID_RUN_HISTORY" }); + }); + + it("consumes the shared strict RFC 3339 corpus for retry availability", async () => { + const corpusPath = fileURLToPath( + new URL("../../../spec/conformance/strict-rfc3339.case.json", import.meta.url), + ); + const corpus = JSON.parse(readFileSync(corpusPath, "utf8")) as { + valid: string[]; + invalid: string[]; + }; + const retrying = graph({ nodes: [node("root", { retry: { maxAttempts: 2 } })] }); + const delegate = new MemoryEventStore(); + const crashStore = new CommitThenThrowStore( + delegate, + (batch) => batch.some((event) => event.type === "NodeRetried"), + ); + await expect(startDurableGraphRun(retrying, {}, { + runId: "date-corpus", + implementationId: "v1", + eventStore: crashStore, + now: fixedNow, + nodeExecutors: { root: () => { throw new Error("retry"); } }, + })).rejects.toBeInstanceOf(DurableRunError); + const base = await history(delegate, "date-corpus"); + const retriedIndex = base.findIndex((event) => event.type === "NodeRetried"); + for (const availableAt of corpus.valid) { + const events = [...base]; + events[retriedIndex] = resign(base[retriedIndex]!, { + ...base[retriedIndex]!.data, availableAt, + }); + const store = await copyHistory("date-corpus", events); + await expect(resumeDurableGraphRun(retrying, { + runId: "date-corpus", + implementationId: "v1", + eventStore: new ConflictStore(store), + }), availableAt).rejects.toMatchObject({ code: "RESUME_CONFLICT" }); + } + for (const availableAt of corpus.invalid) { + const events = [...base]; + events[retriedIndex] = resign(base[retriedIndex]!, { + ...base[retriedIndex]!.data, availableAt, + }); + const store = await copyHistory("date-corpus", events); + await expect(resumeDurableGraphRun(retrying, { + runId: "date-corpus", + implementationId: "v1", + eventStore: store, + }), availableAt).rejects.toMatchObject({ code: "INVALID_RUN_HISTORY" }); + } + }); + + it("latches an unrepresentable retry date and resumes after the clock is corrected", async () => { + const retrying = graph({ + nodes: [node("root", { + retry: { maxAttempts: 2, initialDelayMs: 1, maxDelayMs: 1 }, + })], + }); + const store = new MemoryEventStore(); + let attempts = 0; + await expect(startDurableGraphRun(retrying, {}, { + runId: "clock-boundary", + implementationId: "v1", + eventStore: store, + now: () => new Date("9999-12-31T23:59:59.999Z"), + nodeExecutors: { + root: () => { + attempts += 1; + throw new Error("retry beyond year 9999"); + }, + }, + })).rejects.toMatchObject({ code: "DURABILITY_STORE_FAILED" }); + expect((await history(store, "clock-boundary")).at(-1)?.type).toBe("NodeStarted"); + const result = await resumeDurableGraphRun(retrying, { + runId: "clock-boundary", + implementationId: "v1", + eventStore: store, + now: fixedNow, + nodeExecutors: { + root: (context) => { + attempts += 1; + expect(context.attempt).toBe(2); + return "recovered"; + }, + }, + }); + expect(result.status).toBe("succeeded"); + expect(attempts).toBe(2); + }); + + it("accepts the timer ceiling and rejects the shared oversized fixture before persistence", async () => { + const maximum = 2_147_483_647; + const atLimit = graph({ + nodes: [node("root", { + timeoutMs: maximum, + retry: { maxAttempts: 2, initialDelayMs: maximum, maxDelayMs: maximum }, + })], + policies: { maxDurationMs: maximum }, + }); + const accepted = await startDurableGraphRun(atLimit, {}, { + runId: "timer-limit", + implementationId: "v1", + eventStore: new MemoryEventStore(), + nodeExecutors: { root: () => "immediate" }, + }); + expect(accepted.status).toBe("succeeded"); + + const fixturePath = fileURLToPath( + new URL("../../../spec/conformance/invalid-oversized-timers.graph.json", import.meta.url), + ); + const oversized = JSON.parse(readFileSync(fixturePath, "utf8")) as GraphSpec; + const store = new MemoryEventStore(); + const executor = vi.fn(() => "never"); + const rejected = await startDurableGraphRun(oversized, {}, { + runId: "timer-oversized", + implementationId: "v1", + eventStore: store, + nodeExecutors: { root: executor }, + }); + expect(rejected).toMatchObject({ + status: "failed", + graphHash: null, + failures: [expect.objectContaining({ code: "GE1007_INVALID_GRAPH" })], + }); + expect(await history(store, "timer-oversized")).toEqual([]); + expect(executor).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/runtime/test/scheduler.test.ts b/packages/runtime/test/scheduler.test.ts index d8ed198..3499596 100644 --- a/packages/runtime/test/scheduler.test.ts +++ b/packages/runtime/test/scheduler.test.ts @@ -3,6 +3,10 @@ import { fileURLToPath } from "node:url"; import type { GraphSpec, NodeSpec } from "@graph-engineering/core"; import { describe, expect, it, vi } from "vitest"; import { runGraph, type NodeExecutor } from "../src/index.js"; +import { + runGraphWithJournal, + type SchedulerJournal, +} from "../src/scheduler.js"; function fixture(name: string): GraphSpec { const path = fileURLToPath(new URL(`../../../spec/conformance/${name}`, import.meta.url)); @@ -92,6 +96,21 @@ function wait(milliseconds: number): Promise { return new Promise((resolve) => setTimeout(resolve, milliseconds)); } +function testJournal(overrides: Partial = {}): SchedulerJournal { + return { + beforeAttempt: async ({ node, attempt }) => ({ + runId: "test-run", + attemptId: `test-run/${node.id}/${attempt}`, + activityKey: `activity/${node.id}`, + }), + attemptFailed: async () => undefined, + nodeSucceeded: async () => undefined, + nodeSettledWithoutAttempt: async () => undefined, + runTerminal: async () => undefined, + ...overrides, + }; +} + describe("runGraph", () => { it("executes a diamond concurrently and binds fan-in ports", async () => { let activeBranches = 0; @@ -374,6 +393,69 @@ describe("runGraph", () => { ]); }); + it("does not resolve inherited node or kind executor properties", async () => { + const inherited = graph({ + entrypoints: ["toString"], + outputs: { result: { node: "toString" } }, + nodes: [node("toString", { kind: "agent" })], + }); + const result = await runGraph(inherited, {}, { nodeExecutors: {}, executors: {} }); + expect(result.status).toBe("failed"); + expect(result.failures[0]).toMatchObject({ + code: "EXECUTOR_NOT_FOUND", nodeId: "toString", attempt: 0, + }); + }); + + it("executes an immutable snapshot of the compiled graph", async () => { + const mutableConfig = { value: "original" }; + const document = graph({ nodes: [node("root", { config: mutableConfig })] }); + const running = runGraph(document, {}, { + nodeExecutors: { + root: ({ graph: runtimeGraph }) => { + expect(Object.isFrozen(runtimeGraph)).toBe(true); + expect(Object.isFrozen(runtimeGraph.nodes[0])).toBe(true); + expect(() => { + (runtimeGraph.nodes as NodeSpec[])[0]!.id = "changed"; + }).toThrow(TypeError); + return (runtimeGraph.nodes[0]?.config as { value: string }).value; + }, + }, + }); + mutableConfig.value = "caller-mutated"; + const result = await running; + expect(result.output).toEqual({ result: "original" }); + }); + + it("treats __proto__ output names and target ports as ordinary own JSON keys", async () => { + const outputs = JSON.parse('{"__proto__":{"node":"consumer"}}') as GraphSpec["outputs"]; + let consumerInput: Record | undefined; + const document = graph({ + outputs, + nodes: [node("root"), node("consumer")], + edges: [{ + id: "root-consumer", + from: { node: "root" }, + to: { node: "consumer", port: "__proto__" }, + }], + }); + const result = await runGraph(document, {}, { + nodeExecutors: { + root: () => ({ safe: true }), + consumer: ({ input }) => { + consumerInput = input as Record; + return { received: Object.hasOwn(input as object, "__proto__") }; + }, + }, + }); + expect(result.status).toBe("succeeded"); + expect(Object.hasOwn(consumerInput as object, "__proto__")).toBe(true); + expect(consumerInput?.__proto__).toEqual({ safe: true }); + expect(Object.getPrototypeOf(consumerInput)).toBe(Object.prototype); + expect(Object.hasOwn(result.output as object, "__proto__")).toBe(true); + expect(result.output?.__proto__).toEqual({ received: true }); + expect(Object.getPrototypeOf(result.output)).toBe(Object.prototype); + }); + it("retries within node and global attempt budgets", async () => { let calls = 0; const result = await runGraph( @@ -397,6 +479,234 @@ describe("runGraph", () => { expect(result.nodes[0]).toMatchObject({ attempts: 3, status: "succeeded" }); }); + it("does not journal a retry when the global attempt budget has no capacity", async () => { + const retryAdmissions: boolean[] = []; + const result = await runGraphWithJournal( + graph({ + nodes: [node("root", { retry: { maxAttempts: 2 } })], + policies: { maxTotalAttempts: 1 }, + }), + {}, + { + nodeExecutors: { root: () => { throw new Error("fail once"); } }, + journal: testJournal({ + attemptFailed: async ({ willRetry }) => { retryAdmissions.push(willRetry); }, + }), + }, + ); + + expect(retryAdmissions).toEqual([false]); + expect(result.totalAttempts).toBe(1); + expect(result.nodes[0]).toMatchObject({ + attempts: 1, + status: "failed", + failure: { retryable: false }, + }); + }); + + it("atomically grants only one final retry reservation to parallel failures", async () => { + let firstAttemptStarts = 0; + let releaseFirstAttempts!: () => void; + const firstAttemptsStarted = new Promise((resolve) => { releaseFirstAttempts = resolve; }); + const calls = new Map(); + const retryAdmissions: { nodeId: string; willRetry: boolean }[] = []; + const executor: NodeExecutor = async ({ node: current }) => { + const count = (calls.get(current.id) ?? 0) + 1; + calls.set(current.id, count); + if (count === 1) { + firstAttemptStarts += 1; + if (firstAttemptStarts === 2) releaseFirstAttempts(); + await firstAttemptsStarted; + throw new Error(`first ${current.id}`); + } + return `${current.id}-recovered`; + }; + + const result = await runGraphWithJournal( + graph({ + entrypoints: ["a", "b"], + outputs: { a: { node: "a" }, b: { node: "b" } }, + nodes: [ + node("a", { retry: { maxAttempts: 2 } }), + node("b", { retry: { maxAttempts: 2 } }), + ], + policies: { maxConcurrency: 2, maxTotalAttempts: 3 }, + }), + {}, + { + nodeExecutors: { a: executor, b: executor }, + journal: testJournal({ + attemptFailed: async ({ node: current, willRetry }) => { + retryAdmissions.push({ nodeId: current.id, willRetry }); + }, + }), + }, + ); + + expect([...calls.values()].reduce((sum, count) => sum + count, 0)).toBe(3); + expect(retryAdmissions.filter(({ willRetry }) => willRetry)).toHaveLength(1); + expect(retryAdmissions.filter(({ willRetry }) => !willRetry)).toHaveLength(1); + expect(result.totalAttempts).toBe(3); + }); + + it("measures only globally admitted, journal-committed attempts", async () => { + let historyOpen = 0; + let maximumHistoryOpen = 0; + const executor = vi.fn(async ({ node: current }: Parameters[0]) => { + await wait(5); + return current.id; + }); + const journal = testJournal({ + beforeAttempt: async ({ node: current, attempt }) => { + historyOpen += 1; + maximumHistoryOpen = Math.max(maximumHistoryOpen, historyOpen); + return { + runId: "metric-run", + attemptId: `metric-run/${current.id}/${attempt}`, + activityKey: `activity/${current.id}`, + }; + }, + attemptFailed: async () => { historyOpen -= 1; }, + nodeSucceeded: async () => { historyOpen -= 1; }, + }); + + const result = await runGraphWithJournal( + graph({ + entrypoints: ["a", "b"], + outputs: { a: { node: "a" }, b: { node: "b" } }, + nodes: [node("a"), node("b")], + policies: { maxConcurrency: 2, maxTotalAttempts: 1 }, + }), + {}, + { nodeExecutors: { a: executor, b: executor }, journal }, + ); + + expect(executor).toHaveBeenCalledOnce(); + expect(result.totalAttempts).toBe(1); + expect(result.maxObservedConcurrency).toBe(1); + expect(maximumHistoryOpen).toBe(1); + expect(historyOpen).toBe(0); + }); + + it("commits a failed attempt before releasing its concurrency slot", async () => { + const order: string[] = []; + const journal = testJournal({ + beforeAttempt: async ({ node: current, attempt }) => { + order.push(`${current.id}:started`); + return { + runId: "ordering-run", + attemptId: `ordering-run/${current.id}/${attempt}`, + activityKey: `activity/${current.id}`, + }; + }, + attemptFailed: async ({ node: current }) => { + order.push(`${current.id}:failure-begin`); + await wait(5); + order.push(`${current.id}:failure-commit`); + }, + nodeSucceeded: async ({ node: current }) => { + order.push(`${current.id}:success-commit`); + }, + }); + + await runGraphWithJournal( + graph({ + entrypoints: ["a", "b"], + outputs: { a: { node: "a" }, b: { node: "b" } }, + nodes: [node("a"), node("b")], + policies: { maxConcurrency: 1, maxTotalAttempts: 2 }, + }), + {}, + { + nodeExecutors: { + a: () => { throw new Error("a failed"); }, + b: () => "b succeeded", + }, + journal, + }, + ); + + expect(order.indexOf("a:failure-commit")).toBeLessThan(order.indexOf("b:started")); + expect(order).toEqual([ + "a:started", + "a:failure-begin", + "a:failure-commit", + "b:started", + "b:success-commit", + ]); + }); + + it("counts persisted retry-delay seeds as reserved global attempts", async () => { + await expect( + runGraphWithJournal( + graph({ + entrypoints: ["a", "b"], + outputs: { a: { node: "a" }, b: { node: "b" } }, + nodes: [node("a"), node("b")], + policies: { maxTotalAttempts: 2 }, + }), + {}, + { + initialTotalAttempts: 1, + initialRetryDelaysMs: new Map([["a", 0], ["b", 0]]), + }, + ), + ).rejects.toThrow("initial durable attempts and retry reservations exceed maxTotalAttempts"); + }); + + it("commits zero-attempt settlement before releasing a failed descendant", async () => { + let releaseRootCommit!: () => void; + let rootCommitStarted!: () => void; + const rootCommitGate = new Promise((resolve) => { releaseRootCommit = resolve; }); + const rootCommitEntered = new Promise((resolve) => { rootCommitStarted = resolve; }); + const order: string[] = []; + const child = vi.fn(() => "should-not-run"); + const journal = testJournal({ + nodeSettledWithoutAttempt: async ({ node: current }) => { + order.push(`${current.id}:begin`); + if (current.id === "root") { + rootCommitStarted(); + await rootCommitGate; + } + order.push(`${current.id}:end`); + }, + }); + + const run = runGraphWithJournal( + graph({ + outputs: { result: { node: "child" } }, + nodes: [node("root", { kind: "agent" }), node("child", { kind: "agent" })], + edges: [{ id: "root-child", from: { node: "root" }, to: { node: "child" } }], + }), + {}, + { nodeExecutors: { child }, journal }, + ); + + await rootCommitEntered; + expect(order).toEqual(["root:begin"]); + expect(child).not.toHaveBeenCalled(); + releaseRootCommit(); + const result = await run; + expect(order).toEqual(["root:begin", "root:end", "child:begin", "child:end"]); + expect(result.nodes.map(({ status }) => status)).toEqual(["failed", "skipped"]); + expect(child).not.toHaveBeenCalled(); + }); + + it("keeps the public runGraph enumerable result shape unchanged", async () => { + const result = await runGraph(graph({}), {}, { nodeExecutors: { root: () => "ok" } }); + expect(Object.keys(result).sort()).toEqual([ + "failures", + "graphHash", + "maxObservedConcurrency", + "nodes", + "output", + "status", + "totalAttempts", + ]); + expect("scheduledOrder" in result).toBe(false); + expect("completionOrder" in result).toBe(false); + }); + it("does not hold an attempt concurrency slot during retry backoff", async () => { const completed: string[] = []; let retryCalls = 0; diff --git a/packages/runtime/vitest.config.ts b/packages/runtime/vitest.config.ts index db69004..b25b2bb 100644 --- a/packages/runtime/vitest.config.ts +++ b/packages/runtime/vitest.config.ts @@ -5,6 +5,9 @@ export default defineConfig({ resolve: { alias: { "@graph-engineering/core": fileURLToPath(new URL("../core/src/index.ts", import.meta.url)), + "@graph-engineering/persistence": fileURLToPath( + new URL("../persistence/src/index.ts", import.meta.url), + ), }, }, }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 318ee45..8e9d8b4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -100,6 +100,9 @@ importers: '@graph-engineering/core': specifier: workspace:* version: link:../core + '@graph-engineering/persistence': + specifier: workspace:* + version: link:../persistence devDependencies: '@types/node': specifier: ^22.0.0 diff --git a/python/README.md b/python/README.md index 4670d9c..53c3f0e 100644 --- a/python/README.md +++ b/python/README.md @@ -241,12 +241,67 @@ absolute storage path. They do not provide cross-process locks or distributed writer leases. SQLite, PostgreSQL, object storage, compaction, and multi-process coordination remain follow-up adapters. +## Durable start and resume + +`start_graph_run` creates a new event-sourced run, while `resume_graph_run` +continues an existing non-terminal history. The operations never silently +substitute for one another. Resume reads the original input from `RunCreated`, +checks the graph and caller-supplied implementation identity, and reuses every +committed successful node. + +```python +from graph_engineering import resume_graph_run, start_graph_run +from graph_engineering.persistence import JsonlEventStore + +store = JsonlEventStore(".graph-engineering") +result = await start_graph_run( + graph, + {"seed": 1}, + handlers, + run_id="research-001", + implementation_id="research-handlers@1", + event_store=store, +) + +# In a later process, after confirming the old coordinator has stopped: +result = await resume_graph_run( + graph, + handlers, + run_id="research-001", + implementation_id="research-handlers@1", + event_store=store, +) +``` + +The durable journal commits `NodeScheduled` and `NodeStarted` before calling a +handler. It commits a validated `NodeSucceeded` together with its ordered +`EdgeEmitted` events before releasing dependants. An interrupted attempt remains +charged to both retry budgets. Nodes declared `sideEffects: none` or +`sideEffects: idempotent` can retry; the latter receives the same +`NodeContext.idempotency_key` on every attempt. An omitted or non-idempotent +declaration fails closed with `IN_DOUBT_SIDE_EFFECT` and is not invoked again. + +Inputs, outputs, implementation IDs, and terminal results use tagged Durable +JSON. Non-integer finite doubles are encoded from their exact IEEE-754 bits, so +hashes do not depend on Python or JavaScript decimal rendering. The public +`encode_durable_json`, `decode_durable_json`, and `durable_json_hash` helpers +implement the shared conformance corpus. + +Terminal resume is idempotent: it returns the recorded result without an event, +checkpoint write, or handler call. CAS detects a losing continuation but is not +a distributed lease. Applications must stop the old coordinator before resume, +forward idempotency keys to remote systems, and reconcile ambiguous external +effects. Scheduler checkpoint acceleration and explicit non-idempotent approval +callbacks are not implemented in this slice; correctness comes from replaying +the complete event stream. + ## Current boundary This alpha deliberately focuses on deterministic DAG compilation and execution. It includes bounded concurrency, retry/backoff, per-attempt timeout, -a graph-wide attempt budget, and named source/input/output port binding. Edge -`map` and `condition`, runtime JSON Schema validation, streams, runtime -checkpoint integration/resume, dynamic graph patches, and provider adapters +a graph-wide attempt budget, named source/input/output port binding, and +event-sourced durable continuation. Edge `map` and `condition`, runtime JSON +Schema validation, streams, checkpoint acceleration, dynamic graph patches, +distributed leases, non-idempotent recovery approval, and provider adapters remain follow-up work. Floating-point Graph IR canonicalization is not stable until the shared protocol adopts RFC 8785. diff --git a/python/src/graph_engineering/__init__.py b/python/src/graph_engineering/__init__.py index a51ce79..b0a078e 100644 --- a/python/src/graph_engineering/__init__.py +++ b/python/src/graph_engineering/__init__.py @@ -10,6 +10,18 @@ compile_graph, try_compile_graph, ) +from .durable import ( + DurableRunError, + DurableRunErrorCode, + resume_graph_run, + start_graph_run, +) +from .durable_json import ( + DurableJsonError, + decode_durable_json, + durable_json_hash, + encode_durable_json, +) from .events import GraphEvent from .models import ( EdgeSpec, @@ -67,6 +79,9 @@ "CompiledGraph", "Diagnostic", "DiagnosticCode", + "DurableJsonError", + "DurableRunError", + "DurableRunErrorCode", "EdgeSpec", "Endpoint", "FailureCode", @@ -108,9 +123,14 @@ "canonical_json", "canonical_sha256", "compile_graph", + "decode_durable_json", + "durable_json_hash", + "encode_durable_json", "evaluate_route_selection", "evaluate_settled_barrier", + "resume_graph_run", "run_graph", + "start_graph_run", "try_compile_graph", ] diff --git a/python/src/graph_engineering/compiler.py b/python/src/graph_engineering/compiler.py index 273365c..c21a609 100644 --- a/python/src/graph_engineering/compiler.py +++ b/python/src/graph_engineering/compiler.py @@ -13,6 +13,7 @@ from ._json import normalize_json_strings from .models import EdgeSpec, GraphSpec, NodeSpec +from .portable_json import portable_json_snapshot class DiagnosticCode(StrEnum): @@ -102,13 +103,18 @@ def _unsafe_input_result() -> CompilationResult: def try_compile_graph(graph: GraphSpec | Mapping[str, Any]) -> CompilationResult: """Validate and compile a graph without raising for expected diagnostics.""" - if not isinstance(graph, GraphSpec): - try: - graph = GraphSpec.model_validate(normalize_json_strings(graph)) - except ValidationError as exc: - return _invalid_result(exc) - except Exception: - return _unsafe_input_result() + try: + validated = ( + graph + if isinstance(graph, GraphSpec) + else GraphSpec.model_validate(normalize_json_strings(graph)) + ) + document = validated.model_dump(mode="json", by_alias=True, exclude_unset=True) + graph = GraphSpec.model_validate(portable_json_snapshot(document)) + except ValidationError as exc: + return _invalid_result(exc) + except Exception: + return _unsafe_input_result() diagnostics: list[Diagnostic] = [] duplicate_nodes = _duplicate_values(node.id for node in graph.nodes) diff --git a/python/src/graph_engineering/durable.py b/python/src/graph_engineering/durable.py new file mode 100644 index 0000000..f913c39 --- /dev/null +++ b/python/src/graph_engineering/durable.py @@ -0,0 +1,2130 @@ +"""Event-sourced durable start and resume operations for the native scheduler.""" + +from __future__ import annotations + +import asyncio +import re +from collections.abc import Callable, Iterable, Mapping +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from enum import StrEnum +from types import MappingProxyType +from typing import Any, cast + +from .canonical import canonical_sha256 +from .compiler import ( + CompiledGraph, + Diagnostic, + DiagnosticCode, + GraphCompileError, + compile_graph, +) +from .durable_json import ( + DurableJsonError, + decode_durable_json, + durable_json_hash, + encode_durable_json, +) +from .events import EventType, GraphEvent +from .models import MAX_SAFE_INTEGER, JsonValue, NodeSpec +from .persistence import EventStore, PersistenceError, VersionConflictError +from .portable_json import PortableJsonError, portable_json_snapshot +from .scheduler import ( + AsyncScheduler, + FailureCode, + NodeFailure, + NodeHandler, + NodeResult, + NodeStatus, + RunResult, + RunStatus, + _AttemptIdentity, + _BindingError, +) + +_CONTRACT_VERSION = "scheduler-recovery/v1alpha1" +_GRAPH_REVISION = 1 +_RFC3339 = re.compile( + r"^\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01])" + r"T([01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?" + r"(?:Z|[+-](?:[01]\d|2[0-3]):[0-5]\d)$" +) + +Clock = Callable[[], str] +EventIdFactory = Callable[[str, int], str] + + +class DurableRunErrorCode(StrEnum): + RUN_NOT_FOUND = "RUN_NOT_FOUND" + RUN_ALREADY_EXISTS = "RUN_ALREADY_EXISTS" + GRAPH_HASH_MISMATCH = "GRAPH_HASH_MISMATCH" + INPUT_HASH_MISMATCH = "INPUT_HASH_MISMATCH" + IMPLEMENTATION_MISMATCH = "IMPLEMENTATION_MISMATCH" + INVALID_RUN_HISTORY = "INVALID_RUN_HISTORY" + NODE_EXECUTION_INTERRUPTED = "NODE_EXECUTION_INTERRUPTED" + IN_DOUBT_SIDE_EFFECT = "IN_DOUBT_SIDE_EFFECT" + RESUME_CONFLICT = "RESUME_CONFLICT" + DURABILITY_STORE_FAILED = "DURABILITY_STORE_FAILED" + + +class DurableRunError(Exception): + """A stable operational failure from durable run coordination.""" + + def __init__( + self, + code: DurableRunErrorCode, + run_id: str, + message: str, + details: Mapping[str, object] | None = None, + *, + cause: BaseException | None = None, + ) -> None: + super().__init__(message) + self.code = code + self.run_id = run_id + self.details = MappingProxyType(dict(details or {})) + self.cause = cause + + def to_dict(self) -> dict[str, object]: + return { + "name": type(self).__name__, + "code": self.code.value, + "runId": self.run_id, + "message": str(self), + "details": dict(self.details), + } + + +def _default_clock() -> str: + return datetime.now(UTC).isoformat(timespec="milliseconds").replace("+00:00", "Z") + + +def _default_event_id(run_id: str, sequence: int) -> str: + return f"{run_id}:{sequence}" + + +def _side_effects(node: NodeSpec) -> str: + return node.side_effects or "unspecified" + + +def _failure_document(failure: NodeFailure) -> dict[str, JsonValue]: + if failure.code is FailureCode.OUTPUT_BINDING_FAILED: + if failure.output_name is None: + raise ValueError("output binding failure requires output_name") + output_document: dict[str, JsonValue] = { + "phase": "output", + "code": failure.code.value, + "message": failure.message, + "outputName": failure.output_name, + "nodeId": failure.node_id, + } + if failure.output_port is not None: + output_document["port"] = failure.output_port + return output_document + + document: dict[str, JsonValue] = { + "phase": "execute", + "code": failure.code.value, + "message": failure.message, + "nodeId": failure.node_id, + "attempt": failure.attempt, + "retryable": failure.retryable, + } + if failure.exception_type is not None: + document["causeName"] = failure.exception_type + if failure.upstream_nodes: + document["upstreamNodeIds"] = list(failure.upstream_nodes) + return document + + +def _node_result_document(result: NodeResult) -> dict[str, JsonValue]: + document: dict[str, JsonValue] = { + "nodeId": result.node_id, + "sequence": result.sequence, + "status": result.status.value, + "attempts": result.attempts, + } + if result.input_bound: + document["input"] = result.input + if result.status is NodeStatus.SUCCEEDED: + document["output"] = result.value + if result.failure is not None: + document["failure"] = _failure_document(result.failure) + return document + + +def _run_result_document(result: RunResult) -> dict[str, JsonValue]: + document: dict[str, JsonValue] = { + "status": result.status.value, + "graphHash": result.graph_hash, + "nodes": [_node_result_document(item) for item in result.nodes.values()], + "failures": [_failure_document(item) for item in result.failures], + "scheduledOrder": list(result.scheduled_order), + "completionOrder": list(result.completion_order), + "maxObservedConcurrency": result.max_observed_concurrency, + "totalAttempts": result.total_attempts, + } + if result.outputs is not None: + document["output"] = dict(result.outputs) + return cast(dict[str, JsonValue], portable_json_snapshot(document)) + + +def _invalid_history(run_id: str, message: str, **details: object) -> DurableRunError: + return DurableRunError( + DurableRunErrorCode.INVALID_RUN_HISTORY, + run_id, + message, + details, + ) + + +@dataclass(frozen=True, slots=True) +class _EventDraft: + type: EventType + data: dict[str, JsonValue] + node_id: str | None = None + edge_id: str | None = None + attempt: int | None = None + + +class _DurableJournal: + def __init__( + self, + *, + graph: CompiledGraph, + run_id: str, + store: EventStore, + version: int, + clock: Clock, + event_id_factory: EventIdFactory, + pre_scheduled: Mapping[str, int] | None = None, + activity_keys: Mapping[str, str] | None = None, + prior_event_ids: Iterable[str] = (), + ) -> None: + self.graph = graph + self.run_id = run_id + self.store = store + self.version = version + self.clock = clock + self.event_id_factory = event_id_factory + self.pre_scheduled = dict(pre_scheduled or {}) + self.retry_available_at: dict[str, str] = {} + self._activity_keys = dict(activity_keys or {}) + self._event_ids = set(prior_event_ids) + self._lock = asyncio.Lock() + self._fatal_error: BaseException | None = None + + def _event(self, draft: _EventDraft, sequence: int) -> GraphEvent: + document: dict[str, Any] = { + "apiVersion": "graphengineering.reacher-z.github.io/events/v1alpha1", + "eventId": self.event_id_factory(self.run_id, sequence), + "type": draft.type, + "timestamp": self.clock(), + "runId": self.run_id, + "graphRevision": _GRAPH_REVISION, + "sequence": sequence, + "payloadHash": canonical_sha256(draft.data), + "redacted": True, + "data": draft.data, + } + if draft.node_id is not None: + document["nodeId"] = draft.node_id + if draft.edge_id is not None: + document["edgeId"] = draft.edge_id + if draft.attempt is not None: + document["attempt"] = draft.attempt + return GraphEvent.model_validate(document) + + async def append( + self, + drafts: list[_EventDraft], + *, + conflict_code: DurableRunErrorCode = DurableRunErrorCode.RESUME_CONFLICT, + ) -> tuple[GraphEvent, ...]: + async with self._lock: + return await self._append_locked(drafts, conflict_code=conflict_code) + + async def _append_locked( + self, + drafts: list[_EventDraft], + *, + conflict_code: DurableRunErrorCode, + ) -> tuple[GraphEvent, ...]: + if self._fatal_error is not None: + raise self._fatal_error + expected_version = self.version + try: + events = tuple( + self._event(draft, expected_version + index + 1) + for index, draft in enumerate(drafts) + ) + seen_ids = set(self._event_ids) + duplicate_ids: set[str] = set() + for event in events: + if event.event_id in seen_ids: + duplicate_ids.add(event.event_id) + seen_ids.add(event.event_id) + if duplicate_ids: + raise DurableRunError( + DurableRunErrorCode.DURABILITY_STORE_FAILED, + self.run_id, + "event id factory produced a duplicate durable event identity", + {"eventIds": sorted(duplicate_ids)}, + ) + returned_version = await self.store.append( + self.run_id, + expected_version, + events, + ) + required_version = expected_version + len(events) + if type(returned_version) is not int or returned_version != required_version: + raise DurableRunError( + DurableRunErrorCode.DURABILITY_STORE_FAILED, + self.run_id, + "durable event store returned an inconsistent stream version", + { + "expectedVersion": required_version, + "actualVersion": returned_version, + }, + ) + except VersionConflictError as exc: + error = DurableRunError( + conflict_code, + self.run_id, + "durable event append lost its expected-version comparison", + exc.details, + cause=exc, + ) + self._fatal_error = error + raise error from exc + except PersistenceError as exc: + error = DurableRunError( + DurableRunErrorCode.DURABILITY_STORE_FAILED, + self.run_id, + "durable event append failed", + {"persistenceCode": exc.code.value, **dict(exc.details)}, + cause=exc, + ) + self._fatal_error = error + raise error from exc + except DurableRunError as exc: + self._fatal_error = exc + raise + except Exception as exc: + error = DurableRunError( + DurableRunErrorCode.DURABILITY_STORE_FAILED, + self.run_id, + "durable event append failed", + {"causeName": type(exc).__name__}, + cause=exc, + ) + self._fatal_error = error + raise error from exc + except BaseException as exc: + self._fatal_error = exc + raise + self.version = returned_version + self._event_ids.update(event.event_id for event in events) + return events + + def activity_key(self, node_id: str, input_hash: str) -> str: + return durable_json_hash( + ["activity/v1alpha1", self.run_id, _GRAPH_REVISION, node_id, input_hash] + ) + + async def before_attempt( + self, + node: NodeSpec, + node_input: JsonValue, + attempt: int, + ) -> _AttemptIdentity: + tagged_input = encode_durable_json(node_input) + input_hash = durable_json_hash(node_input) + activity_key = self.activity_key(node.id, input_hash) + self._activity_keys[node.id] = activity_key + drafts: list[_EventDraft] = [] + if self.pre_scheduled.pop(node.id, None) != attempt: + drafts.append( + _EventDraft( + "NodeScheduled", + { + "input": tagged_input, + "inputHash": input_hash, + "activityKey": activity_key, + "sideEffects": _side_effects(node), + }, + node_id=node.id, + attempt=attempt, + ) + ) + drafts.append( + _EventDraft( + "NodeStarted", + {"inputHash": input_hash, "activityKey": activity_key}, + node_id=node.id, + attempt=attempt, + ) + ) + await self.append(drafts) + return _AttemptIdentity( + run_id=self.run_id, + attempt_id=f"{self.run_id}/{node.id}/{attempt}", + activity_key=activity_key, + ) + + async def attempt_failed( + self, + failure: NodeFailure, + *, + will_retry: bool, + retry_delay_ms: float, + ) -> int: + async with self._lock: + if self._fatal_error is not None: + raise self._fatal_error + available_at: str | None = None + try: + drafts = [ + _EventDraft( + "NodeAttemptFailed", + { + "terminal": not will_retry, + "failure": _failure_document(failure), + }, + node_id=failure.node_id, + attempt=failure.attempt, + ) + ] + if will_retry: + node = self.graph.nodes[failure.node_id] + activity_key = self._activity_keys.get(failure.node_id, "") + if not activity_key: + raise _invalid_history( + self.run_id, + "cannot record a retry without its scheduled recovery identity", + nodeId=node.id, + attempt=failure.attempt, + ) + available_at = _add_milliseconds(self.clock(), retry_delay_ms) + drafts.append( + _EventDraft( + "NodeRetried", + {"availableAt": available_at, "activityKey": activity_key}, + node_id=failure.node_id, + attempt=failure.attempt + 1, + ) + ) + except DurableRunError as exc: + self._fatal_error = exc + raise + except Exception as exc: + error = DurableRunError( + DurableRunErrorCode.DURABILITY_STORE_FAILED, + self.run_id, + "durable retry event could not be prepared", + {"causeName": type(exc).__name__}, + cause=exc, + ) + self._fatal_error = error + raise error from exc + except BaseException as exc: + self._fatal_error = exc + raise + events = await self._append_locked( + drafts, + conflict_code=DurableRunErrorCode.RESUME_CONFLICT, + ) + if available_at is not None: + self.retry_available_at[failure.node_id] = available_at + return events[0].sequence + + async def node_succeeded( + self, + node: NodeSpec, + node_input: JsonValue, + attempt: int, + output: JsonValue, + ) -> int: + input_hash = durable_json_hash(node_input) + tagged_output = encode_durable_json(output) + output_hash = durable_json_hash(output) + drafts = [ + _EventDraft( + "NodeSucceeded", + { + "inputHash": input_hash, + "output": tagged_output, + "outputHash": output_hash, + }, + node_id=node.id, + attempt=attempt, + ) + ] + drafts.extend( + _EventDraft( + "EdgeEmitted", + {"outputHash": output_hash}, + node_id=node.id, + edge_id=edge.id, + attempt=attempt, + ) + for edge in sorted(self.graph.outgoing[node.id], key=lambda item: item.id) + ) + events = await self.append(drafts) + return events[0].sequence + + async def node_settled_without_attempt( + self, + node: NodeSpec, + result: NodeResult, + ) -> int: + events = await self.append( + [ + _EventDraft( + "NodeSettledWithoutAttempt", + {"result": encode_durable_json(_node_result_document(result))}, + node_id=node.id, + ) + ] + ) + return events[0].sequence + + async def run_terminal(self, result: RunResult) -> None: + event_type = cast( + EventType, + { + RunStatus.SUCCEEDED: "RunSucceeded", + RunStatus.FAILED: "RunFailed", + RunStatus.CANCELLED: "RunCancelled", + }[result.status], + ) + await self.append( + [_EventDraft(event_type, {"result": encode_durable_json(_run_result_document(result))})] + ) + + +def _strict_rfc3339(value: str) -> datetime: + if not _RFC3339.fullmatch(value): + raise ValueError("timestamp must be strict RFC 3339 with a UTC offset") + normalized = f"{value[:-1]}+00:00" if value.endswith("Z") else value + parsed = datetime.fromisoformat(normalized) + if parsed.tzinfo is None or parsed.utcoffset() is None: + raise ValueError("timestamp must include a UTC offset") + return parsed + + +def _add_milliseconds(value: str, milliseconds: float) -> str: + parsed = _strict_rfc3339(value) + timedelta(milliseconds=milliseconds) + return parsed.isoformat(timespec="milliseconds").replace("+00:00", "Z") + + +def _object(value: object, run_id: str, context: str) -> dict[str, JsonValue]: + if type(value) is not dict: + raise _invalid_history(run_id, f"{context} must be an object") + return cast(dict[str, JsonValue], value) + + +def _array(value: object, run_id: str, context: str) -> list[JsonValue]: + if type(value) is not list: + raise _invalid_history(run_id, f"{context} must be an array") + return cast(list[JsonValue], value) + + +def _string(value: object, run_id: str, context: str) -> str: + if type(value) is not str: + raise _invalid_history(run_id, f"{context} must be a string") + return value + + +def _integer(value: object, run_id: str, context: str, *, minimum: int = 0) -> int: + if type(value) is not int or value < minimum or value > MAX_SAFE_INTEGER: + raise _invalid_history(run_id, f"{context} must be a safe integer >= {minimum}") + return value + + +def _exact_keys( + value: Mapping[str, object], + expected: set[str], + run_id: str, + context: str, +) -> None: + if set(value) != expected: + raise _invalid_history( + run_id, + f"{context} has unexpected fields", + expected=sorted(expected), + actual=sorted(value), + ) + + +def _decode_failure(value: object, run_id: str, context: str) -> NodeFailure: + document = _object(value, run_id, context) + phase = _string(document.get("phase"), run_id, f"{context}.phase") + if phase == "output": + required = {"phase", "code", "message", "outputName", "nodeId"} + allowed = required | {"port"} + if not required.issubset(document) or not set(document).issubset(allowed): + raise _invalid_history(run_id, f"{context} has invalid output-failure fields") + if document["code"] != FailureCode.OUTPUT_BINDING_FAILED.value: + raise _invalid_history(run_id, f"{context} has invalid output-failure code") + return NodeFailure( + code=FailureCode.OUTPUT_BINDING_FAILED, + message=_string(document["message"], run_id, f"{context}.message"), + node_id=_string(document["nodeId"], run_id, f"{context}.nodeId"), + attempt=0, + output_name=_string(document["outputName"], run_id, f"{context}.outputName"), + output_port=( + _string(document["port"], run_id, f"{context}.port") if "port" in document else None + ), + ) + if phase != "execute": + raise _invalid_history(run_id, f"{context}.phase must be 'execute' or 'output'") + required = {"phase", "code", "message", "nodeId", "attempt", "retryable"} + allowed = required | {"causeName", "upstreamNodeIds"} + if not required.issubset(document) or not set(document).issubset(allowed): + raise _invalid_history(run_id, f"{context} has invalid failure fields") + try: + code = FailureCode(_string(document["code"], run_id, f"{context}.code")) + except ValueError as exc: + raise _invalid_history(run_id, f"{context}.code is unknown") from exc + if code is FailureCode.OUTPUT_BINDING_FAILED: + raise _invalid_history(run_id, f"{context} uses an output code in execute phase") + retryable = document["retryable"] + if type(retryable) is not bool: + raise _invalid_history(run_id, f"{context}.retryable must be a boolean") + upstream: tuple[str, ...] = () + if "upstreamNodeIds" in document: + upstream = tuple( + _string(item, run_id, f"{context}.upstreamNodeIds") + for item in _array(document["upstreamNodeIds"], run_id, context) + ) + if len(upstream) != len(set(upstream)): + raise _invalid_history(run_id, f"{context}.upstreamNodeIds contains duplicates") + if code is FailureCode.UPSTREAM_FAILED: + if not upstream: + raise _invalid_history(run_id, f"{context} omits failed upstream nodes") + elif "upstreamNodeIds" in document: + raise _invalid_history(run_id, f"{context} has upstream nodes for a non-upstream failure") + attempt = _integer(document["attempt"], run_id, f"{context}.attempt") + return NodeFailure( + code=code, + message=_string(document["message"], run_id, f"{context}.message"), + node_id=_string(document["nodeId"], run_id, f"{context}.nodeId"), + attempt=attempt, + retryable=retryable, + exception_type=( + _string(document["causeName"], run_id, f"{context}.causeName") + if "causeName" in document + else None + ), + upstream_nodes=upstream, + ) + + +def _decode_node_result(value: object, run_id: str, context: str) -> NodeResult: + document = _object(value, run_id, context) + required = {"nodeId", "sequence", "status", "attempts"} + allowed = required | {"input", "output", "failure"} + if not required.issubset(document) or not set(document).issubset(allowed): + raise _invalid_history(run_id, f"{context} has invalid node-result fields") + try: + status = NodeStatus(_string(document["status"], run_id, f"{context}.status")) + except ValueError as exc: + raise _invalid_history(run_id, f"{context}.status is unknown") from exc + node_id = _string(document["nodeId"], run_id, f"{context}.nodeId") + attempts = _integer(document["attempts"], run_id, f"{context}.attempts") + failure = ( + _decode_failure(document["failure"], run_id, f"{context}.failure") + if "failure" in document + else None + ) + has_output = "output" in document + if status is NodeStatus.SUCCEEDED: + if not has_output or failure is not None: + raise _invalid_history(run_id, f"{context} has an incoherent successful outcome") + else: + if has_output or failure is None: + raise _invalid_history(run_id, f"{context} has an incoherent terminal failure") + if failure.retryable: + raise _invalid_history(run_id, f"{context} hides a retryable terminal outcome") + if failure.node_id != node_id or failure.attempt != attempts: + raise _invalid_history(run_id, f"{context} failure identity is inconsistent") + if failure.code is FailureCode.OUTPUT_BINDING_FAILED: + raise _invalid_history(run_id, f"{context} contains an output failure as a node result") + skipped_codes = {FailureCode.UPSTREAM_FAILED, FailureCode.NODE_CANCELLED} + if status is NodeStatus.SKIPPED and failure.code not in skipped_codes: + raise _invalid_history(run_id, f"{context} has an invalid skipped-node failure") + if status is NodeStatus.FAILED and failure.code is FailureCode.UPSTREAM_FAILED: + raise _invalid_history(run_id, f"{context} reports an upstream skip as failed") + return NodeResult( + node_id=node_id, + sequence=_integer(document["sequence"], run_id, f"{context}.sequence"), + status=status, + attempts=attempts, + input=document.get("input"), + value=document["output"] if has_output else None, + failure=failure, + input_bound="input" in document, + ) + + +def _decode_run_result(value: object, run_id: str, graph: CompiledGraph) -> RunResult: + document = _object(value, run_id, "terminal result") + required = { + "status", + "graphHash", + "nodes", + "failures", + "scheduledOrder", + "completionOrder", + "maxObservedConcurrency", + "totalAttempts", + } + if not required.issubset(document) or not set(document).issubset(required | {"output"}): + raise _invalid_history(run_id, "terminal result has invalid fields") + try: + status = RunStatus(_string(document["status"], run_id, "terminal result.status")) + except ValueError as exc: + raise _invalid_history(run_id, "terminal result status is unknown") from exc + graph_hash = _string(document["graphHash"], run_id, "terminal result.graphHash") + if graph_hash != graph.graph_hash: + raise _invalid_history(run_id, "terminal result graph hash is inconsistent") + node_items = [ + _decode_node_result(item, run_id, f"terminal result.nodes[{index}]") + for index, item in enumerate(_array(document["nodes"], run_id, "terminal nodes")) + ] + if [item.node_id for item in node_items] != list(graph.topological_order): + raise _invalid_history(run_id, "terminal result nodes are not in topological order") + failures = tuple( + _decode_failure(item, run_id, f"terminal result.failures[{index}]") + for index, item in enumerate(_array(document["failures"], run_id, "terminal failures")) + ) + output: Mapping[str, JsonValue] | None = None + if "output" in document: + output = MappingProxyType(_object(document["output"], run_id, "terminal output")) + return RunResult( + status=status, + graph_hash=graph_hash, + nodes=MappingProxyType({item.node_id: item for item in node_items}), + outputs=output, + failures=failures, + scheduled_order=tuple( + _string(item, run_id, "terminal scheduled order") + for item in _array(document["scheduledOrder"], run_id, "terminal scheduled order") + ), + completion_order=tuple( + _string(item, run_id, "terminal completion order") + for item in _array(document["completionOrder"], run_id, "terminal completion order") + ), + max_observed_concurrency=_integer( + document["maxObservedConcurrency"], + run_id, + "terminal maxObservedConcurrency", + ), + total_attempts=_integer(document["totalAttempts"], run_id, "terminal totalAttempts"), + ) + + +@dataclass(slots=True) +class _NodeProjection: + node_id: str + input: JsonValue = None + input_bound: bool = False + input_hash: str | None = None + activity_key: str | None = None + side_effects: str | None = None + scheduled_attempt: int | None = None + open_attempt: int | None = None + retry_attempt: int | None = None + retry_available_at: str | None = None + attempts: int = 0 + result: NodeResult | None = None + output_hash: str | None = None + + +@dataclass(frozen=True, slots=True) +class _FoldedRun: + graph_input: JsonValue + graph_hash: str + input_hash: str + implementation_hash: str + max_total_attempts: int + projections: Mapping[str, _NodeProjection] + total_attempts: int + scheduled_order: tuple[str, ...] + completion_order: tuple[str, ...] + max_observed_concurrency: int + terminal_result: RunResult | None + version: int + event_ids: frozenset[str] + + +def _terminal_outputs( + graph: CompiledGraph, + nodes: Mapping[str, NodeResult], +) -> tuple[Mapping[str, JsonValue] | None, tuple[NodeFailure, ...]]: + output_values: dict[str, JsonValue] = {} + failures: list[NodeFailure] = [] + complete = True + for output_name in sorted(graph.spec.outputs): + endpoint = graph.spec.outputs[output_name] + result = nodes[endpoint.node] + if result.status is not NodeStatus.SUCCEEDED: + complete = False + continue + value = result.value + if endpoint.port is not None: + if type(value) is not dict or endpoint.port not in value: + complete = False + message = f"output from {endpoint.node!r} does not contain port {endpoint.port!r}" + failures.append( + NodeFailure( + FailureCode.OUTPUT_BINDING_FAILED, + f"could not bind graph output {output_name!r}: {message}", + endpoint.node, + 0, + output_name=output_name, + output_port=endpoint.port, + ) + ) + continue + value = value[endpoint.port] + output_values[output_name] = portable_json_snapshot(value) + return ( + MappingProxyType(output_values) if complete else None, + tuple(failures), + ) + + +def _same_failure_semantics(left: NodeFailure, right: NodeFailure) -> bool: + """Compare portable failure facts while ignoring language-specific prose.""" + + return ( + left.code is right.code + and left.node_id == right.node_id + and left.attempt == right.attempt + and left.retryable is right.retryable + and left.upstream_nodes == right.upstream_nodes + and left.output_name == right.output_name + and left.output_port == right.output_port + ) + + +def _same_node_result_semantics(left: NodeResult, right: NodeResult) -> bool: + if ( + left.node_id != right.node_id + or left.sequence != right.sequence + or left.status is not right.status + or left.attempts != right.attempts + or left.input_bound is not right.input_bound + or (left.input_bound and left.input != right.input) + or left.value != right.value + or (left.failure is None) != (right.failure is None) + ): + return False + return left.failure is None or _same_failure_semantics( + left.failure, cast(NodeFailure, right.failure) + ) + + +def _same_failure_sequence( + left: tuple[NodeFailure, ...], + right: tuple[NodeFailure, ...], +) -> bool: + return len(left) == len(right) and all( + _same_failure_semantics(item, expected) for item, expected in zip(left, right, strict=True) + ) + + +def _validate_terminal_result( + *, + run_id: str, + event_type: EventType, + terminal: RunResult, + graph: CompiledGraph, + projections: Mapping[str, _NodeProjection], + folded_scheduled: tuple[str, ...], + folded_completion: tuple[str, ...], + max_observed_concurrency: int, + total_attempts: int, +) -> None: + expected_status = { + "RunSucceeded": RunStatus.SUCCEEDED, + "RunFailed": RunStatus.FAILED, + "RunCancelled": RunStatus.CANCELLED, + }[event_type] + if terminal.status is not expected_status: + raise _invalid_history(run_id, "terminal event and result status disagree") + if terminal.total_attempts != total_attempts: + raise _invalid_history(run_id, "terminal result attempt count is inconsistent") + if terminal.max_observed_concurrency != max_observed_concurrency: + raise _invalid_history(run_id, "terminal max concurrency contradicts event history") + if any(item.open_attempt is not None for item in projections.values()): + raise _invalid_history(run_id, "terminal event leaves an attempt open") + + expected_nodes: dict[str, NodeResult] = {} + for index, node_id in enumerate(graph.topological_order): + terminal_node = terminal.nodes[node_id] + projection = projections[node_id] + if terminal_node.sequence != index or terminal_node.attempts != projection.attempts: + raise _invalid_history( + run_id, + "terminal node sequence or attempt count contradicts history", + nodeId=node_id, + ) + if projection.result is None: + raise _invalid_history( + run_id, + "terminal result contains a node with no committed outcome", + nodeId=node_id, + ) + if not _same_node_result_semantics(terminal_node, projection.result): + raise _invalid_history( + run_id, + "terminal result contradicts folded node history", + nodeId=node_id, + ) + expected_nodes[node_id] = projection.result + + expected_outputs, output_failures = _terminal_outputs(graph, expected_nodes) + if (terminal.outputs is None) != (expected_outputs is None) or ( + terminal.outputs is not None + and dict(terminal.outputs) != dict(cast(Mapping[str, JsonValue], expected_outputs)) + ): + raise _invalid_history(run_id, "terminal output contradicts committed node outputs") + expected_failures = ( + tuple(result.failure for result in expected_nodes.values() if result.failure is not None) + + output_failures + ) + if not _same_failure_sequence(terminal.failures, expected_failures): + raise _invalid_history(run_id, "terminal failures contradict folded node outcomes") + + node_ids = set(graph.nodes) + for name, order in ( + ("scheduledOrder", terminal.scheduled_order), + ("completionOrder", terminal.completion_order), + ): + if len(order) != len(node_ids) or set(order) != node_ids: + raise _invalid_history(run_id, f"terminal {name} is not a node permutation") + if ( + terminal.scheduled_order != folded_scheduled + or terminal.completion_order != folded_completion + ): + raise _invalid_history(run_id, "terminal node orders contradict folded history") + + succeeded = ( + not expected_failures + and expected_outputs is not None + and all(item.status is NodeStatus.SUCCEEDED for item in expected_nodes.values()) + ) + if terminal.status is RunStatus.SUCCEEDED and not succeeded: + raise _invalid_history(run_id, "terminal status contradicts reconstructed graph result") + if terminal.status is RunStatus.FAILED and succeeded: + raise _invalid_history(run_id, "terminal status contradicts reconstructed graph result") + + +def _event_attempt(event: GraphEvent, run_id: str) -> int: + if event.attempt is None: + raise _invalid_history(run_id, f"{event.type} requires an attempt") + return event.attempt + + +def _event_node(event: GraphEvent, graph: CompiledGraph, run_id: str) -> str: + if event.node_id is None or event.node_id not in graph.nodes: + raise _invalid_history(run_id, f"{event.type} references an unknown node") + return event.node_id + + +def _max_node_attempts(node: NodeSpec) -> int: + return node.retry.max_attempts if node.retry and node.retry.max_attempts else 1 + + +def _bound_input_from_history( + graph: CompiledGraph, + node_id: str, + graph_input: JsonValue, + projections: Mapping[str, _NodeProjection], + run_id: str, +) -> JsonValue: + incoming = sorted(graph.incoming[node_id], key=lambda edge: edge.id) + if not incoming: + return portable_json_snapshot(graph_input) + values: dict[str, JsonValue] = {} + for edge in incoming: + source = projections[edge.source.node].result + if source is None or source.status is not NodeStatus.SUCCEEDED: + raise _invalid_history( + run_id, + "NodeScheduled was emitted before every dependency succeeded", + nodeId=node_id, + sourceNodeId=edge.source.node, + ) + value = source.value + if edge.source.port is not None: + if type(value) is not dict or edge.source.port not in value: + raise _invalid_history( + run_id, + "NodeScheduled source port is absent from committed output", + nodeId=node_id, + sourceNodeId=edge.source.node, + port=edge.source.port, + ) + value = value[edge.source.port] + key = edge.target.port or edge.source.node + if key in values: + raise _invalid_history( + run_id, + "NodeScheduled input binding contains a duplicate target key", + nodeId=node_id, + key=key, + ) + values[key] = value + return portable_json_snapshot(values) + + +def _validate_settled_without_attempt( + *, + run_id: str, + graph: CompiledGraph, + graph_input: JsonValue, + projections: Mapping[str, _NodeProjection], + projection: _NodeProjection, + result: NodeResult, + total_attempts: int, + max_total_attempts: int, + retry_reservations: set[str], +) -> None: + node_id = projection.node_id + sequence = graph.topological_order.index(node_id) + if ( + result.node_id != node_id + or result.sequence != sequence + or result.attempts != projection.attempts + or result.status is NodeStatus.SUCCEEDED + or result.failure is None + or result.failure.node_id != node_id + or result.failure.attempt != projection.attempts + ): + raise _invalid_history( + run_id, + "NodeSettledWithoutAttempt result identity is invalid", + nodeId=node_id, + ) + + committed = { + item_id: item.result for item_id, item in projections.items() if item.result is not None + } + incoming = graph.incoming[node_id] + unresolved = sorted( + {edge.source.node for edge in incoming if edge.source.node not in committed} + ) + if unresolved: + raise _invalid_history( + run_id, + "NodeSettledWithoutAttempt was emitted before every dependency settled", + nodeId=node_id, + unresolvedUpstreamNodeIds=unresolved, + ) + failed_upstream = tuple( + sorted( + { + edge.source.node + for edge in incoming + if committed[edge.source.node].status is not NodeStatus.SUCCEEDED + } + ) + ) + + expected: NodeResult | None = None + failure = result.failure + if failure.code is FailureCode.NODE_CANCELLED: + if result.status is NodeStatus.SKIPPED: + expected = NodeResult( + node_id=node_id, + sequence=sequence, + status=NodeStatus.SKIPPED, + attempts=projection.attempts, + failure=NodeFailure( + FailureCode.NODE_CANCELLED, + "node did not start because the run was cancelled", + node_id, + projection.attempts, + ), + input_bound=False, + ) + else: + try: + node_input = AsyncScheduler._bind_input( + graph, + node_id, + graph_input, + cast(Mapping[str, NodeResult], committed), + ) + except _BindingError as exc: + raise _invalid_history( + run_id, + "cancelled node did not have bindable input", + nodeId=node_id, + ) from exc + expected = NodeResult( + node_id=node_id, + sequence=sequence, + status=NodeStatus.FAILED, + attempts=projection.attempts, + input=node_input, + failure=NodeFailure( + FailureCode.NODE_CANCELLED, + f"node {node_id!r} was cancelled", + node_id, + projection.attempts, + ), + ) + elif failed_upstream: + expected = NodeResult( + node_id=node_id, + sequence=sequence, + status=NodeStatus.SKIPPED, + attempts=projection.attempts, + failure=NodeFailure( + FailureCode.UPSTREAM_FAILED, + "node did not start because upstream dependencies failed", + node_id, + projection.attempts, + upstream_nodes=failed_upstream, + ), + input_bound=False, + ) + else: + try: + node_input = AsyncScheduler._bind_input( + graph, + node_id, + graph_input, + cast(Mapping[str, NodeResult], committed), + ) + except _BindingError as exc: + expected = NodeResult( + node_id=node_id, + sequence=sequence, + status=NodeStatus.FAILED, + attempts=projection.attempts, + failure=NodeFailure( + FailureCode.INPUT_BINDING_FAILED, + f"could not bind input for node {node_id!r}: {exc}", + node_id, + projection.attempts, + exception_type=type(exc).__name__, + ), + input_bound=False, + ) + else: + node = graph.nodes[node_id] + if failure.code is FailureCode.EXECUTOR_NOT_FOUND: + expected_failure = NodeFailure( + FailureCode.EXECUTOR_NOT_FOUND, + f"no executor is registered for node {node_id!r} or kind {node.kind!r}", + node_id, + projection.attempts, + ) + elif projection.attempts >= _max_node_attempts(node): + expected_failure = NodeFailure( + FailureCode.ATTEMPT_BUDGET_EXHAUSTED, + f"node {node_id!r} has exhausted its durable attempt budget", + node_id, + projection.attempts, + ) + elif ( + node_id not in retry_reservations + and total_attempts + len(retry_reservations) >= max_total_attempts + ): + expected_failure = NodeFailure( + FailureCode.ATTEMPT_BUDGET_EXHAUSTED, + f"run attempt budget was exhausted before node {node_id!r} could start", + node_id, + projection.attempts, + ) + else: + expected_failure = None + if expected_failure is not None: + expected = NodeResult( + node_id=node_id, + sequence=sequence, + status=NodeStatus.FAILED, + attempts=projection.attempts, + input=node_input, + failure=expected_failure, + ) + + if expected is None or not _same_node_result_semantics(result, expected): + raise _invalid_history( + run_id, + "NodeSettledWithoutAttempt result is not scheduler-derived", + nodeId=node_id, + code=failure.code.value, + ) + + +def _fold_history( + events: tuple[GraphEvent, ...], + *, + graph: CompiledGraph, + run_id: str, + implementation_hash: str, +) -> _FoldedRun: + if not events: + raise DurableRunError( + DurableRunErrorCode.RUN_NOT_FOUND, + run_id, + "durable run does not exist", + ) + projections = {node_id: _NodeProjection(node_id) for node_id in graph.nodes} + graph_input: JsonValue = None + stored_graph_hash = "" + stored_input_hash = "" + stored_implementation_hash = "" + max_total_attempts = 0 + started = False + terminal_result: RunResult | None = None + terminal_seen = False + scheduled_order: list[str] = [] + completion_order: list[str] = [] + active: set[str] = set() + max_observed = 0 + total_attempts = 0 + retry_reservations: set[str] = set() + expected_edges: list[tuple[str, str, int, str]] = [] + expected_retry: tuple[str, int, str] | None = None + declaration_order = {node_id: index for index, node_id in enumerate(graph.nodes)} + event_ids: set[str] = set() + + for index, event in enumerate(events): + if event.sequence != index or event.run_id != run_id: + raise _invalid_history(run_id, "event identity or sequence is inconsistent") + try: + _strict_rfc3339(event.timestamp) + except ValueError as exc: + raise _invalid_history( + run_id, + "event timestamp is not strict RFC 3339", + sequence=index, + ) from exc + if event.graph_revision != _GRAPH_REVISION: + raise _invalid_history(run_id, "event graph revision is not 1", sequence=index) + if event.event_id in event_ids: + raise _invalid_history( + run_id, + "eventId is duplicated", + eventId=event.event_id, + ) + event_ids.add(event.event_id) + if event.payload_hash is None or event.payload_hash != canonical_sha256(event.data): + raise _invalid_history(run_id, "event payload hash is invalid", sequence=index) + if terminal_seen: + raise _invalid_history(run_id, "event appears after a terminal event", sequence=index) + if event.type in { + "RunCreated", + "RunStarted", + "RunResumed", + "RunSucceeded", + "RunFailed", + "RunCancelled", + } and (event.node_id is not None or event.edge_id is not None or event.attempt is not None): + raise _invalid_history( + run_id, + f"{event.type} has unexpected node, edge, or attempt identity", + ) + if ( + event.type + in { + "NodeScheduled", + "NodeStarted", + "NodeSucceeded", + "NodeAttemptFailed", + "NodeRetried", + "NodeSettledWithoutAttempt", + } + and event.edge_id is not None + ): + raise _invalid_history(run_id, f"{event.type} has an edge identity") + if event.type == "NodeSettledWithoutAttempt" and event.attempt is not None: + raise _invalid_history( + run_id, + "NodeSettledWithoutAttempt has an attempt identity", + ) + + if expected_edges: + edge_id, producer_id, attempt, output_hash = expected_edges.pop(0) + if event.type != "EdgeEmitted" or event.edge_id != edge_id: + raise _invalid_history( + run_id, + "NodeSucceeded is not followed by its ordered EdgeEmitted batch", + sequence=index, + expectedEdgeId=edge_id, + ) + if event.node_id != producer_id or event.attempt != attempt: + raise _invalid_history(run_id, "EdgeEmitted producer identity is invalid") + _exact_keys(event.data, {"outputHash"}, run_id, "EdgeEmitted.data") + if event.data["outputHash"] != output_hash: + raise _invalid_history(run_id, "EdgeEmitted output hash is invalid") + continue + + if expected_retry is not None: + node_id, attempt, activity_key = expected_retry + if event.type != "NodeRetried" or event.node_id != node_id or event.attempt != attempt: + raise _invalid_history( + run_id, + "retryable NodeAttemptFailed is not followed by NodeRetried", + sequence=index, + ) + _exact_keys( + event.data, + {"availableAt", "activityKey"}, + run_id, + "NodeRetried.data", + ) + available_at = _string(event.data["availableAt"], run_id, "availableAt") + try: + _strict_rfc3339(available_at) + except ValueError as exc: + raise _invalid_history(run_id, "NodeRetried.availableAt is invalid") from exc + if event.data["activityKey"] != activity_key: + raise _invalid_history(run_id, "NodeRetried activity key is invalid") + projection = projections[node_id] + if attempt > _max_node_attempts(graph.nodes[node_id]): + raise _invalid_history( + run_id, + "NodeRetried exceeds the node retry budget", + nodeId=node_id, + attempt=attempt, + ) + if node_id in retry_reservations: + raise _invalid_history(run_id, "NodeRetried duplicates a retry reservation") + if total_attempts + len(retry_reservations) >= max_total_attempts: + raise _invalid_history( + run_id, + "NodeRetried is not eligible under the global attempt budget", + nodeId=node_id, + attempt=attempt, + ) + projection.retry_attempt = attempt + projection.retry_available_at = available_at + retry_reservations.add(node_id) + expected_retry = None + continue + + if index == 0: + if event.type != "RunCreated": + raise _invalid_history(run_id, "RunCreated must be the first event") + _exact_keys( + event.data, + { + "contractVersion", + "graphHash", + "implementationHash", + "input", + "inputHash", + "maxTotalAttempts", + }, + run_id, + "RunCreated.data", + ) + if event.data["contractVersion"] != _CONTRACT_VERSION: + raise _invalid_history(run_id, "RunCreated contract version is incompatible") + stored_graph_hash = _string(event.data["graphHash"], run_id, "graphHash") + stored_implementation_hash = _string( + event.data["implementationHash"], run_id, "implementationHash" + ) + stored_input_hash = _string(event.data["inputHash"], run_id, "inputHash") + max_total_attempts = _integer( + event.data["maxTotalAttempts"], + run_id, + "maxTotalAttempts", + minimum=1, + ) + try: + graph_input = decode_durable_json(event.data["input"]) + except DurableJsonError as exc: + raise DurableRunError( + DurableRunErrorCode.INPUT_HASH_MISMATCH, + run_id, + "stored graph input is not valid Durable JSON", + cause=exc, + ) from exc + if durable_json_hash(graph_input) != stored_input_hash: + raise DurableRunError( + DurableRunErrorCode.INPUT_HASH_MISMATCH, + run_id, + "stored graph input does not match inputHash", + ) + if stored_graph_hash != graph.graph_hash: + raise DurableRunError( + DurableRunErrorCode.GRAPH_HASH_MISMATCH, + run_id, + "compiled graph does not match the durable run", + {"expected": stored_graph_hash, "actual": graph.graph_hash}, + ) + if stored_implementation_hash != implementation_hash: + raise DurableRunError( + DurableRunErrorCode.IMPLEMENTATION_MISMATCH, + run_id, + "implementationId does not match the durable run", + {"expected": stored_implementation_hash, "actual": implementation_hash}, + ) + effective_limit = ( + graph.spec.policies.max_total_attempts + if graph.spec.policies and graph.spec.policies.max_total_attempts is not None + else MAX_SAFE_INTEGER + ) + if max_total_attempts != effective_limit: + raise _invalid_history(run_id, "stored attempt budget is inconsistent with graph") + continue + + if event.type == "RunCreated": + raise _invalid_history(run_id, "RunCreated appears more than once", sequence=index) + if event.type == "RunStarted": + if started or index != 1 or event.data: + raise _invalid_history(run_id, "RunStarted is duplicate, misplaced, or non-empty") + started = True + continue + if not started: + raise _invalid_history(run_id, "lifecycle event appears before RunStarted") + + if event.type == "RunResumed": + _exact_keys( + event.data, + {"reusedNodeIds", "interruptedNodeIds"}, + run_id, + "RunResumed.data", + ) + resume_lists: dict[str, list[str]] = {} + for field_name in ("reusedNodeIds", "interruptedNodeIds"): + node_ids = [ + _string(item, run_id, field_name) + for item in _array(event.data[field_name], run_id, field_name) + ] + if any(node_id not in graph.nodes for node_id in node_ids): + raise _invalid_history(run_id, f"{field_name} contains an unknown node") + if node_ids != sorted(node_ids, key=declaration_order.__getitem__): + raise _invalid_history( + run_id, + f"{field_name} is not in graph declaration order", + ) + if len(node_ids) != len(set(node_ids)): + raise _invalid_history(run_id, f"{field_name} contains duplicates") + resume_lists[field_name] = node_ids + if set(resume_lists["reusedNodeIds"]) & set(resume_lists["interruptedNodeIds"]): + raise _invalid_history(run_id, "RunResumed node lists overlap") + expected_reused: list[str] = [] + for node_id in graph.nodes: + projected_result = projections[node_id].result + if projected_result is not None and projected_result.status is NodeStatus.SUCCEEDED: + expected_reused.append(node_id) + expected_interrupted = [ + node_id for node_id in graph.nodes if projections[node_id].open_attempt is not None + ] + if resume_lists["reusedNodeIds"] != expected_reused: + raise _invalid_history(run_id, "RunResumed reusedNodeIds contradict history") + if resume_lists["interruptedNodeIds"] != expected_interrupted: + raise _invalid_history( + run_id, + "RunResumed interruptedNodeIds contradict history", + ) + continue + + if event.type == "NodeSettledWithoutAttempt": + node_id = _event_node(event, graph, run_id) + projection = projections[node_id] + if projection.open_attempt is not None or projection.result is not None: + raise _invalid_history( + run_id, + "NodeSettledWithoutAttempt targets an open or settled node", + nodeId=node_id, + ) + _exact_keys( + event.data, + {"result"}, + run_id, + "NodeSettledWithoutAttempt.data", + ) + try: + decoded_result = decode_durable_json(event.data["result"]) + except DurableJsonError as exc: + raise _invalid_history( + run_id, + "NodeSettledWithoutAttempt result is malformed", + ) from exc + result = _decode_node_result( + decoded_result, + run_id, + "NodeSettledWithoutAttempt.result", + ) + _validate_settled_without_attempt( + run_id=run_id, + graph=graph, + graph_input=graph_input, + projections=projections, + projection=projection, + result=result, + total_attempts=total_attempts, + max_total_attempts=max_total_attempts, + retry_reservations=retry_reservations, + ) + retry_reservations.discard(node_id) + projection.retry_attempt = None + projection.retry_available_at = None + projection.result = result + if node_id not in scheduled_order: + scheduled_order.append(node_id) + if node_id not in completion_order: + completion_order.append(node_id) + continue + + if event.type == "NodeScheduled": + node_id = _event_node(event, graph, run_id) + attempt = _event_attempt(event, run_id) + projection = projections[node_id] + if projection.result is not None or projection.open_attempt is not None: + raise _invalid_history(run_id, "settled or running node was scheduled again") + if projection.scheduled_attempt is not None and projection.retry_attempt is None: + raise _invalid_history(run_id, "NodeScheduled is duplicated without a retry") + owns_reservation = node_id in retry_reservations + if (projection.retry_attempt is not None) is not owns_reservation: + raise _invalid_history( + run_id, + "NodeScheduled retry reservation is inconsistent", + nodeId=node_id, + ) + expected_attempt = projection.retry_attempt or projection.attempts + 1 + if attempt != expected_attempt: + raise _invalid_history(run_id, "NodeScheduled attempt is non-contiguous") + if attempt > _max_node_attempts(graph.nodes[node_id]): + raise _invalid_history( + run_id, + "NodeScheduled exceeds the node retry budget", + nodeId=node_id, + attempt=attempt, + ) + if ( + not owns_reservation + and total_attempts + len(retry_reservations) >= max_total_attempts + ): + raise _invalid_history( + run_id, + "NodeScheduled is not eligible under the global attempt budget", + nodeId=node_id, + attempt=attempt, + ) + _exact_keys( + event.data, + {"input", "inputHash", "activityKey", "sideEffects"}, + run_id, + "NodeScheduled.data", + ) + try: + node_input = decode_durable_json(event.data["input"]) + except DurableJsonError as exc: + raise _invalid_history(run_id, "NodeScheduled input is malformed") from exc + input_hash = _string(event.data["inputHash"], run_id, "inputHash") + if durable_json_hash(node_input) != input_hash: + raise _invalid_history(run_id, "NodeScheduled inputHash is invalid") + expected_input = _bound_input_from_history( + graph, + node_id, + graph_input, + projections, + run_id, + ) + if node_input != expected_input: + raise _invalid_history( + run_id, + "NodeScheduled input does not match deterministic graph binding", + nodeId=node_id, + attempt=attempt, + ) + if projection.attempts > 0 and node_input != projection.input: + raise _invalid_history( + run_id, + "retry changed the original bound node input", + nodeId=node_id, + attempt=attempt, + ) + activity_key = _string(event.data["activityKey"], run_id, "activityKey") + expected_activity = durable_json_hash( + ["activity/v1alpha1", run_id, _GRAPH_REVISION, node_id, input_hash] + ) + if activity_key != expected_activity: + raise _invalid_history(run_id, "NodeScheduled activityKey is invalid") + side_effects = _string(event.data["sideEffects"], run_id, "sideEffects") + if side_effects != _side_effects(graph.nodes[node_id]): + raise _invalid_history(run_id, "NodeScheduled sideEffects is inconsistent") + projection.input = node_input + projection.input_bound = True + projection.input_hash = input_hash + projection.activity_key = activity_key + projection.side_effects = side_effects + projection.scheduled_attempt = attempt + projection.retry_attempt = None + if node_id not in scheduled_order: + scheduled_order.append(node_id) + continue + + if event.type == "NodeStarted": + node_id = _event_node(event, graph, run_id) + attempt = _event_attempt(event, run_id) + projection = projections[node_id] + if ( + projection.scheduled_attempt != attempt + or projection.open_attempt is not None + or projection.result is not None + or projection.retry_attempt is not None + ): + raise _invalid_history(run_id, "NodeStarted lacks its required NodeScheduled") + _exact_keys(event.data, {"inputHash", "activityKey"}, run_id, "NodeStarted.data") + if ( + event.data["inputHash"] != projection.input_hash + or event.data["activityKey"] != projection.activity_key + ): + raise _invalid_history(run_id, "NodeStarted recovery identity is invalid") + if attempt > _max_node_attempts(graph.nodes[node_id]): + raise _invalid_history( + run_id, + "NodeStarted exceeds the node retry budget", + nodeId=node_id, + attempt=attempt, + ) + if node_id in retry_reservations: + retry_reservations.remove(node_id) + elif total_attempts + len(retry_reservations) >= max_total_attempts: + raise _invalid_history( + run_id, + "NodeStarted is not eligible under the global attempt budget", + nodeId=node_id, + attempt=attempt, + ) + projection.open_attempt = attempt + projection.retry_available_at = None + projection.scheduled_attempt = None + projection.attempts = attempt + total_attempts += 1 + if total_attempts + len(retry_reservations) > max_total_attempts: + raise _invalid_history(run_id, "history exceeds the global attempt budget") + active.add(node_id) + policy_concurrency = ( + graph.spec.policies.max_concurrency if graph.spec.policies is not None else None + ) + if policy_concurrency is not None and len(active) > policy_concurrency: + raise _invalid_history( + run_id, + "history exceeds the graph concurrency policy", + maxConcurrency=policy_concurrency, + activeNodeIds=sorted(active, key=declaration_order.__getitem__), + ) + max_observed = max(max_observed, len(active)) + continue + + if event.type == "NodeSucceeded": + node_id = _event_node(event, graph, run_id) + attempt = _event_attempt(event, run_id) + projection = projections[node_id] + if projection.open_attempt != attempt or projection.result is not None: + raise _invalid_history(run_id, "NodeSucceeded lacks one open attempt") + _exact_keys( + event.data, + {"inputHash", "output", "outputHash"}, + run_id, + "NodeSucceeded.data", + ) + if event.data["inputHash"] != projection.input_hash: + raise _invalid_history(run_id, "NodeSucceeded inputHash is invalid") + try: + output = decode_durable_json(event.data["output"]) + except DurableJsonError as exc: + raise _invalid_history(run_id, "NodeSucceeded output is malformed") from exc + output_hash = _string(event.data["outputHash"], run_id, "outputHash") + if durable_json_hash(output) != output_hash: + raise _invalid_history(run_id, "NodeSucceeded outputHash is invalid") + projection.output_hash = output_hash + projection.open_attempt = None + active.discard(node_id) + projection.result = NodeResult( + node_id=node_id, + sequence=graph.topological_order.index(node_id), + status=NodeStatus.SUCCEEDED, + attempts=attempt, + input=projection.input, + value=output, + input_bound=projection.input_bound, + ) + if node_id not in completion_order: + completion_order.append(node_id) + expected_edges = [ + (edge.id, node_id, attempt, output_hash) + for edge in sorted(graph.outgoing[node_id], key=lambda item: item.id) + ] + continue + + if event.type == "NodeAttemptFailed": + node_id = _event_node(event, graph, run_id) + attempt = _event_attempt(event, run_id) + projection = projections[node_id] + if projection.open_attempt != attempt or projection.result is not None: + raise _invalid_history(run_id, "NodeAttemptFailed lacks one open attempt") + _exact_keys( + event.data, + {"terminal", "failure"}, + run_id, + "NodeAttemptFailed.data", + ) + terminal = event.data["terminal"] + if type(terminal) is not bool: + raise _invalid_history(run_id, "NodeAttemptFailed.terminal must be boolean") + failure = _decode_failure(event.data["failure"], run_id, "attempt failure") + if failure.node_id != node_id or failure.attempt != attempt: + raise _invalid_history(run_id, "attempt failure identity is invalid") + allowed_attempt_codes = { + FailureCode.NODE_EXECUTION_FAILED, + FailureCode.NODE_TIMEOUT, + FailureCode.NODE_CANCELLED, + FailureCode.INVALID_OUTPUT, + FailureCode.NODE_EXECUTION_INTERRUPTED, + } + if failure.code not in allowed_attempt_codes: + raise _invalid_history( + run_id, + "NodeAttemptFailed uses a non-attempt outcome code", + nodeId=node_id, + code=failure.code.value, + ) + if node_id in retry_reservations: + raise _invalid_history( + run_id, + "open attempt retained a retry reservation", + nodeId=node_id, + ) + retryable_codes = { + FailureCode.NODE_EXECUTION_FAILED, + FailureCode.NODE_TIMEOUT, + FailureCode.INVALID_OUTPUT, + FailureCode.NODE_EXECUTION_INTERRUPTED, + } + retry_is_safe = ( + failure.code is not FailureCode.NODE_EXECUTION_INTERRUPTED + or projection.side_effects in {"none", "idempotent"} + ) + should_retry = ( + failure.code in retryable_codes + and retry_is_safe + and attempt < _max_node_attempts(graph.nodes[node_id]) + and total_attempts + len(retry_reservations) < max_total_attempts + ) + if terminal is should_retry or failure.retryable is not should_retry: + raise _invalid_history( + run_id, + "attempt retryability contradicts deterministic eligibility", + nodeId=node_id, + attempt=attempt, + expectedRetryable=should_retry, + ) + if failure.code is FailureCode.NODE_EXECUTION_INTERRUPTED and ( + failure.message != "process ended before the attempt outcome was durably recorded" + or failure.exception_type != "ProcessLost" + ): + raise _invalid_history( + run_id, + "interrupted attempt failure is not canonical", + nodeId=node_id, + attempt=attempt, + ) + if failure.code is FailureCode.NODE_CANCELLED and (not terminal or failure.retryable): + raise _invalid_history( + run_id, + "cancelled attempt failure is not canonical", + nodeId=node_id, + attempt=attempt, + ) + projection.open_attempt = None + active.discard(node_id) + if terminal: + projection.result = NodeResult( + node_id=node_id, + sequence=graph.topological_order.index(node_id), + status=NodeStatus.FAILED, + attempts=attempt, + input=projection.input, + failure=failure, + input_bound=projection.input_bound, + ) + if node_id not in completion_order: + completion_order.append(node_id) + else: + if projection.activity_key is None: + raise _invalid_history(run_id, "retry has no activity key") + expected_retry = (node_id, attempt + 1, projection.activity_key) + continue + + if event.type in {"RunSucceeded", "RunFailed", "RunCancelled"}: + if retry_reservations: + raise _invalid_history( + run_id, + "terminal event leaves retry reservations pending", + nodeIds=sorted(retry_reservations, key=declaration_order.__getitem__), + ) + _exact_keys(event.data, {"result"}, run_id, f"{event.type}.data") + try: + decoded_result = decode_durable_json(event.data["result"]) + except DurableJsonError as exc: + raise _invalid_history(run_id, "terminal result is malformed") from exc + terminal_result = _decode_run_result(decoded_result, run_id, graph) + _validate_terminal_result( + run_id=run_id, + event_type=event.type, + terminal=terminal_result, + graph=graph, + projections=projections, + folded_scheduled=tuple(scheduled_order), + folded_completion=tuple(completion_order), + max_observed_concurrency=max_observed, + total_attempts=total_attempts, + ) + terminal_seen = True + continue + + raise _invalid_history( + run_id, + f"event type {event.type!r} is outside durable DAG recovery", + sequence=index, + ) + + if expected_edges: + raise _invalid_history(run_id, "event stream ends inside an EdgeEmitted batch") + if expected_retry is not None: + raise _invalid_history(run_id, "event stream ends before required NodeRetried") + if not started: + raise _invalid_history(run_id, "event stream omits RunStarted") + return _FoldedRun( + graph_input=graph_input, + graph_hash=stored_graph_hash, + input_hash=stored_input_hash, + implementation_hash=stored_implementation_hash, + max_total_attempts=max_total_attempts, + projections=MappingProxyType(projections), + total_attempts=total_attempts, + scheduled_order=tuple(scheduled_order), + completion_order=tuple(completion_order), + max_observed_concurrency=max_observed, + terminal_result=terminal_result, + version=events[-1].sequence, + event_ids=frozenset(event_ids), + ) + + +async def _read_history(store: EventStore, run_id: str) -> tuple[GraphEvent, ...]: + try: + return await store.read(run_id) + except PersistenceError: + # Corrupt bytes and invalid envelopes retain their persistence-layer + # identity, as required by the recovery contract. + raise + except Exception as exc: + raise DurableRunError( + DurableRunErrorCode.DURABILITY_STORE_FAILED, + run_id, + "durable event history could not be read", + {"causeName": type(exc).__name__}, + cause=exc, + ) from exc + + +def _implementation_hash(implementation_id: str) -> str: + if type(implementation_id) is not str or not implementation_id: + raise ValueError("implementation_id must be a non-empty string") + return durable_json_hash(implementation_id) + + +def _fresh_compiled_graph(graph: CompiledGraph) -> CompiledGraph: + if not isinstance(graph, CompiledGraph): + raise TypeError("graph must be a CompiledGraph") + try: + document = graph.spec.model_dump( + mode="json", + by_alias=True, + exclude_unset=True, + ) + snapshot = portable_json_snapshot(document) + except Exception as exc: + raise GraphCompileError( + ( + Diagnostic( + DiagnosticCode.INVALID_GRAPH, + "compiled graph could not be snapshotted as portable JSON", + ), + ) + ) from exc + if type(snapshot) is not dict: + raise GraphCompileError( + ( + Diagnostic( + DiagnosticCode.INVALID_GRAPH, + "compiled graph snapshot is not an object", + ), + ) + ) + return compile_graph(cast(Mapping[str, Any], snapshot)) + + +def _effective_attempt_limit(graph: CompiledGraph) -> int: + if graph.spec.policies and graph.spec.policies.max_total_attempts is not None: + return graph.spec.policies.max_total_attempts + return MAX_SAFE_INTEGER + + +def _remaining_delay(available_at: str, now: str) -> float: + return max( + 0.0, + (_strict_rfc3339(available_at) - _strict_rfc3339(now)).total_seconds(), + ) + + +async def start_graph_run( + graph: CompiledGraph, + graph_input: JsonValue, + handlers: Mapping[str, NodeHandler] | None = None, + *, + run_id: str, + implementation_id: str, + event_store: EventStore, + max_concurrency: int | None = None, + cancel_event: asyncio.Event | None = None, + clock: Clock | None = None, + event_id_factory: EventIdFactory | None = None, +) -> RunResult: + """Create and execute a new durable run; never silently resume one.""" + + handler_snapshot = dict(handlers or {}) + graph = _fresh_compiled_graph(graph) + implementation_hash = _implementation_hash(implementation_id) + try: + input_snapshot = portable_json_snapshot(graph_input) + except PortableJsonError: + raise TypeError("graph input must be a portable finite JSON value") from None + if await _read_history(event_store, run_id): + raise DurableRunError( + DurableRunErrorCode.RUN_ALREADY_EXISTS, + run_id, + "durable run already exists", + ) + input_hash = durable_json_hash(input_snapshot) + journal = _DurableJournal( + graph=graph, + run_id=run_id, + store=event_store, + version=-1, + clock=clock or _default_clock, + event_id_factory=event_id_factory or _default_event_id, + ) + await journal.append( + [ + _EventDraft( + "RunCreated", + { + "contractVersion": _CONTRACT_VERSION, + "graphHash": graph.graph_hash, + "implementationHash": implementation_hash, + "input": encode_durable_json(input_snapshot), + "inputHash": input_hash, + "maxTotalAttempts": _effective_attempt_limit(graph), + }, + ), + _EventDraft("RunStarted", {}), + ], + conflict_code=DurableRunErrorCode.RUN_ALREADY_EXISTS, + ) + scheduler = AsyncScheduler(handler_snapshot, max_concurrency=max_concurrency) + return await scheduler._run( + graph, + input_snapshot, + cancel_event=cancel_event, + journal=journal, + ) + + +async def resume_graph_run( + graph: CompiledGraph, + handlers: Mapping[str, NodeHandler] | None = None, + *, + run_id: str, + implementation_id: str, + event_store: EventStore, + max_concurrency: int | None = None, + cancel_event: asyncio.Event | None = None, + clock: Clock | None = None, + event_id_factory: EventIdFactory | None = None, +) -> RunResult: + """Resume one non-terminal durable history without replacing its input.""" + + handler_snapshot = dict(handlers or {}) + graph = _fresh_compiled_graph(graph) + implementation_hash = _implementation_hash(implementation_id) + events = await _read_history(event_store, run_id) + folded = _fold_history( + events, + graph=graph, + run_id=run_id, + implementation_hash=implementation_hash, + ) + if folded.terminal_result is not None: + return folded.terminal_result + for node_id, projection in folded.projections.items(): + if ( + projection.result is not None + and projection.result.failure is not None + and projection.result.failure.code is FailureCode.NODE_EXECUTION_INTERRUPTED + and projection.side_effects not in {"none", "idempotent"} + ): + raise DurableRunError( + DurableRunErrorCode.IN_DOUBT_SIDE_EFFECT, + run_id, + "an interrupted node remains blocked on unsafe side-effect reconciliation", + { + "nodeId": node_id, + "attempt": projection.result.attempts, + "sideEffects": projection.side_effects or "unspecified", + }, + ) + + effective_clock = clock or _default_clock + journal = _DurableJournal( + graph=graph, + run_id=run_id, + store=event_store, + version=folded.version, + clock=effective_clock, + event_id_factory=event_id_factory or _default_event_id, + pre_scheduled={ + node_id: projection.scheduled_attempt + for node_id, projection in folded.projections.items() + if projection.scheduled_attempt is not None + and projection.open_attempt is None + and projection.retry_attempt is None + and projection.result is None + }, + activity_keys={ + node_id: projection.activity_key + for node_id, projection in folded.projections.items() + if projection.activity_key is not None + }, + prior_event_ids=folded.event_ids, + ) + declaration_order = {node_id: index for index, node_id in enumerate(graph.nodes)} + interrupted = sorted( + [ + node_id + for node_id, projection in folded.projections.items() + if projection.open_attempt is not None + ], + key=declaration_order.__getitem__, + ) + reused = sorted( + [ + node_id + for node_id, projection in folded.projections.items() + if projection.result is not None and projection.result.status is NodeStatus.SUCCEEDED + ], + key=declaration_order.__getitem__, + ) + await journal.append( + [ + _EventDraft( + "RunResumed", + { + "reusedNodeIds": [cast(JsonValue, item) for item in reused], + "interruptedNodeIds": [cast(JsonValue, item) for item in interrupted], + }, + ) + ] + ) + initial_results = { + node_id: projection.result + for node_id, projection in folded.projections.items() + if projection.result is not None + } + total_attempts = folded.total_attempts + retry_delays = { + node_id: _remaining_delay( + projection.retry_available_at, + effective_clock(), + ) + for node_id, projection in folded.projections.items() + if ( + projection.retry_available_at is not None + and projection.open_attempt is None + and projection.result is None + ) + } + initial_completion_order = list(folded.completion_order) + + for node_id in interrupted: + projection = folded.projections[node_id] + attempt = cast(int, projection.open_attempt) + side_effects = projection.side_effects or "unspecified" + node = graph.nodes[node_id] + max_attempts = node.retry.max_attempts if node.retry and node.retry.max_attempts else 1 + automatic = side_effects in {"none", "idempotent"} + can_retry = ( + automatic + and attempt < max_attempts + and total_attempts + len(retry_delays) < folded.max_total_attempts + ) + interrupted_failure = NodeFailure( + FailureCode.NODE_EXECUTION_INTERRUPTED, + "process ended before the attempt outcome was durably recorded", + node_id, + attempt, + retryable=can_retry, + exception_type="ProcessLost", + ) + retry_delay_ms = AsyncScheduler._retry_delay_ms(node, attempt) if can_retry else 0.0 + await journal.attempt_failed( + interrupted_failure, + will_retry=can_retry, + retry_delay_ms=retry_delay_ms, + ) + if not automatic: + raise DurableRunError( + DurableRunErrorCode.IN_DOUBT_SIDE_EFFECT, + run_id, + "an interrupted node may have produced an unsafe external effect", + {"nodeId": node_id, "attempt": attempt, "sideEffects": side_effects}, + ) + if not can_retry: + initial_results[node_id] = NodeResult( + node_id=node_id, + sequence=graph.topological_order.index(node_id), + status=NodeStatus.FAILED, + attempts=attempt, + input=projection.input, + failure=interrupted_failure, + input_bound=projection.input_bound, + ) + if node_id not in initial_completion_order: + initial_completion_order.append(node_id) + else: + available_at = journal.retry_available_at[node_id] + retry_delays[node_id] = _remaining_delay(available_at, effective_clock()) + + scheduler = AsyncScheduler(handler_snapshot, max_concurrency=max_concurrency) + return await scheduler._run( + graph, + folded.graph_input, + cancel_event=cancel_event, + journal=journal, + initial_results=cast(Mapping[str, NodeResult], initial_results), + attempt_offsets={ + node_id: projection.attempts for node_id, projection in folded.projections.items() + }, + initial_total_attempts=total_attempts, + initial_scheduled_order=folded.scheduled_order, + initial_completion_order=tuple(initial_completion_order), + initial_max_observed_concurrency=folded.max_observed_concurrency, + initial_retry_delays=retry_delays, + ) diff --git a/python/src/graph_engineering/durable_json.py b/python/src/graph_engineering/durable_json.py new file mode 100644 index 0000000..9ceee3d --- /dev/null +++ b/python/src/graph_engineering/durable_json.py @@ -0,0 +1,126 @@ +"""Tagged, cross-language Durable JSON encoding and hashing.""" + +from __future__ import annotations + +import math +import re +import struct +from typing import cast + +from ._json import normalize_json_string +from .canonical import canonical_sha256 +from .models import MAX_SAFE_INTEGER, JsonValue +from .portable_json import PortableJsonError, portable_json_snapshot + +_FLOAT_BITS = re.compile(r"^[0-9a-f]{16}$") + + +class DurableJsonError(ValueError): + """Raised when tagged Durable JSON is malformed or non-canonical.""" + + +def _encode(value: JsonValue) -> JsonValue: + if value is None: + return ["n"] + if type(value) is bool: + return ["b", value] + if type(value) is str: + return ["s", value] + if type(value) is int: + return ["i", value] + if type(value) is float: + return ["f", struct.pack(">d", value).hex()] + if type(value) is list: + return ["a", [_encode(item) for item in value]] + if type(value) is dict: + return [ + "o", + [[key, _encode(value[key])] for key in sorted(value)], + ] + raise AssertionError("portable JSON snapshot returned an unsupported value") + + +def encode_durable_json(value: object) -> JsonValue: + """Detach and encode one portable finite JSON value.""" + + try: + snapshot = portable_json_snapshot(value) + except PortableJsonError as exc: + raise DurableJsonError(str(exc)) from exc + return _encode(snapshot) + + +def _tagged_array(value: object, *, path: str) -> list[JsonValue]: + if type(value) is not list or not value or type(value[0]) is not str: + raise DurableJsonError(f"{path}: expected a non-empty tagged array") + return cast(list[JsonValue], value) + + +def _decode(value: object, *, path: str) -> JsonValue: + tagged = _tagged_array(value, path=path) + tag = tagged[0] + if tag == "n": + if len(tagged) != 1: + raise DurableJsonError(f"{path}: tag 'n' requires arity 1") + return None + if tag == "b": + if len(tagged) != 2 or type(tagged[1]) is not bool: + raise DurableJsonError(f"{path}: tag 'b' requires one boolean payload") + return tagged[1] + if tag == "s": + if len(tagged) != 2 or type(tagged[1]) is not str: + raise DurableJsonError(f"{path}: tag 's' requires one string payload") + return normalize_json_string(tagged[1]) + if tag == "i": + if ( + len(tagged) != 2 + or type(tagged[1]) is not int + or abs(tagged[1]) > MAX_SAFE_INTEGER + ): + raise DurableJsonError(f"{path}: tag 'i' requires one safe-integer payload") + return tagged[1] + if tag == "f": + bits = tagged[1] if len(tagged) == 2 else None + if type(bits) is not str or _FLOAT_BITS.fullmatch(bits) is None: + raise DurableJsonError(f"{path}: tag 'f' requires sixteen lowercase hex digits") + decoded = float(struct.unpack(">d", bytes.fromhex(bits))[0]) + if not math.isfinite(decoded) or decoded.is_integer(): + raise DurableJsonError( + f"{path}: tag 'f' must encode a finite non-integer binary64 value" + ) + return decoded + if tag == "a": + payload = tagged[1] if len(tagged) == 2 else None + if type(payload) is not list: + raise DurableJsonError(f"{path}: tag 'a' requires one array payload") + return [_decode(item, path=f"{path}/1/{index}") for index, item in enumerate(payload)] + if tag == "o": + payload = tagged[1] if len(tagged) == 2 else None + if type(payload) is not list: + raise DurableJsonError(f"{path}: tag 'o' requires one entries-array payload") + result: dict[str, JsonValue] = {} + previous: str | None = None + for index, entry in enumerate(payload): + if type(entry) is not list or len(entry) != 2 or type(entry[0]) is not str: + raise DurableJsonError(f"{path}/1/{index}: expected [string, tagged-value]") + key = normalize_json_string(entry[0]) + if previous is not None and key <= previous: + raise DurableJsonError( + f"{path}/1/{index}/0: object keys must be unique and code-point sorted" + ) + previous = key + result[key] = _decode(entry[1], path=f"{path}/1/{index}/1") + return result + raise DurableJsonError(f"{path}: unknown Durable JSON tag {tag!r}") + + +def decode_durable_json(value: object) -> JsonValue: + """Decode and strictly validate one canonical tagged Durable JSON value.""" + + return _decode(value, path="#") + + +def durable_json_hash(value: object) -> str: + """Hash the canonical tagged representation of one portable JSON value.""" + + return canonical_sha256(encode_durable_json(value)) diff --git a/python/src/graph_engineering/events.py b/python/src/graph_engineering/events.py index 2cf14ae..9baecb6 100644 --- a/python/src/graph_engineering/events.py +++ b/python/src/graph_engineering/events.py @@ -21,6 +21,7 @@ "NodeScheduled", "NodeStarted", "NodeAttemptFailed", + "NodeSettledWithoutAttempt", "NodeRetried", "NodeSucceeded", "EdgeEmitted", @@ -36,7 +37,8 @@ "RunSucceeded", ] _RFC3339 = re.compile( - r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})$" + r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?" + r"(?:Z|[+-](?:[01]\d|2[0-3]):[0-5]\d)$" ) @@ -69,9 +71,7 @@ class GraphEvent(StrictModel): type: EventType timestamp: str run_id: Annotated[str, Field(min_length=1)] = Field(alias="runId") - graph_revision: Annotated[int, Field(ge=1, le=MAX_SAFE_INTEGER)] = Field( - alias="graphRevision" - ) + graph_revision: Annotated[int, Field(ge=1, le=MAX_SAFE_INTEGER)] = Field(alias="graphRevision") sequence: Annotated[int, Field(ge=0, le=MAX_SAFE_INTEGER)] trace_id: str | None = Field(default=None, alias="traceId") span_id: str | None = Field(default=None, alias="spanId") diff --git a/python/src/graph_engineering/models.py b/python/src/graph_engineering/models.py index e06de1c..1c6084e 100644 --- a/python/src/graph_engineering/models.py +++ b/python/src/graph_engineering/models.py @@ -2,6 +2,7 @@ from __future__ import annotations +import math from typing import Annotated, Literal, TypeAlias from pydantic import BaseModel, ConfigDict, Field, field_validator @@ -11,6 +12,11 @@ JsonValue: TypeAlias = PydanticJsonValue JsonObject: TypeAlias = dict[str, JsonValue] MAX_SAFE_INTEGER = 2**53 - 1 +MAX_TIMER_MILLISECONDS = 2**31 - 1 +SafePositiveInteger: TypeAlias = Annotated[int, Field(ge=1, le=MAX_SAFE_INTEGER)] +SafeNonNegativeInteger: TypeAlias = Annotated[int, Field(ge=0, le=MAX_SAFE_INTEGER)] +TimerPositiveMilliseconds: TypeAlias = Annotated[int, Field(ge=1, le=MAX_TIMER_MILLISECONDS)] +TimerNonNegativeMilliseconds: TypeAlias = Annotated[int, Field(ge=0, le=MAX_TIMER_MILLISECONDS)] NodeKind: TypeAlias = Literal[ "agent", @@ -56,15 +62,24 @@ class RetryPolicy(StrictModel): max_attempts: Annotated[int, Field(ge=1, le=100)] | None = Field( default=None, alias="maxAttempts" ) - initial_delay_ms: Annotated[int, Field(ge=0)] | None = Field( + initial_delay_ms: TimerNonNegativeMilliseconds | None = Field( default=None, alias="initialDelayMs" ) - max_delay_ms: Annotated[int, Field(ge=0)] | None = Field(default=None, alias="maxDelayMs") + max_delay_ms: TimerNonNegativeMilliseconds | None = Field(default=None, alias="maxDelayMs") backoff_multiplier: Annotated[float, Field(ge=1)] | None = Field( default=None, alias="backoffMultiplier" ) jitter: bool | None = None + @field_validator("backoff_multiplier") + @classmethod + def backoff_multiplier_is_portable(cls, value: float | None) -> float | None: + if value is not None and ( + not math.isfinite(value) or (value.is_integer() and abs(value) > MAX_SAFE_INTEGER) + ): + raise ValueError("backoffMultiplier must be a portable finite number") + return value + class NodeSpec(StrictModel): id: Identifier @@ -73,7 +88,7 @@ class NodeSpec(StrictModel): output_schema: JsonObject = Field(alias="outputSchema") config: JsonValue retry: RetryPolicy | None = None - timeout_ms: Annotated[int, Field(ge=1)] | None = Field(default=None, alias="timeoutMs") + timeout_ms: TimerPositiveMilliseconds | None = Field(default=None, alias="timeoutMs") cache: JsonObject | None = None resources: JsonObject | None = None isolation: JsonObject | None = None @@ -100,22 +115,23 @@ class GraphPolicies(BaseModel): strict=True, ) - max_concurrency: Annotated[int, Field(ge=1)] | None = Field( - default=None, alias="maxConcurrency" - ) - max_dynamic_nodes: Annotated[int, Field(ge=0)] | None = Field( - default=None, alias="maxDynamicNodes" - ) - max_depth: Annotated[int, Field(ge=1)] | None = Field(default=None, alias="maxDepth") - max_fan_out: Annotated[int, Field(ge=1)] | None = Field(default=None, alias="maxFanOut") - max_total_attempts: Annotated[int, Field(ge=1)] | None = Field( - default=None, alias="maxTotalAttempts" - ) - max_duration_ms: Annotated[int, Field(ge=1)] | None = Field( - default=None, alias="maxDurationMs" - ) + max_concurrency: SafePositiveInteger | None = Field(default=None, alias="maxConcurrency") + max_dynamic_nodes: SafeNonNegativeInteger | None = Field(default=None, alias="maxDynamicNodes") + max_depth: SafePositiveInteger | None = Field(default=None, alias="maxDepth") + max_fan_out: SafePositiveInteger | None = Field(default=None, alias="maxFanOut") + max_total_attempts: SafePositiveInteger | None = Field(default=None, alias="maxTotalAttempts") + max_duration_ms: TimerPositiveMilliseconds | None = Field(default=None, alias="maxDurationMs") max_cost_usd: Annotated[float, Field(ge=0)] | None = Field(default=None, alias="maxCostUsd") + @field_validator("max_cost_usd") + @classmethod + def max_cost_usd_is_portable(cls, value: float | None) -> float | None: + if value is not None and ( + not math.isfinite(value) or (value.is_integer() and abs(value) > MAX_SAFE_INTEGER) + ): + raise ValueError("maxCostUsd must be a portable finite number") + return value + class GraphSpec(StrictModel): api_version: Literal["graphengineering.reacher-z.github.io/v1alpha1"] = Field( diff --git a/python/src/graph_engineering/scheduler.py b/python/src/graph_engineering/scheduler.py index 37732cb..fdd405a 100644 --- a/python/src/graph_engineering/scheduler.py +++ b/python/src/graph_engineering/scheduler.py @@ -5,12 +5,14 @@ import asyncio import inspect from collections.abc import Awaitable, Callable, Mapping +from contextlib import suppress from dataclasses import dataclass from enum import StrEnum from types import MappingProxyType +from typing import Protocol -from .compiler import CompiledGraph -from .models import Endpoint, GraphSpec, JsonValue, NodeSpec +from .compiler import CompiledGraph, compile_graph +from .models import MAX_SAFE_INTEGER, Endpoint, GraphSpec, JsonValue, NodeSpec from .portable_json import PortableJsonError, portable_json_snapshot @@ -36,6 +38,7 @@ class FailureCode(StrEnum): INPUT_BINDING_FAILED = "INPUT_BINDING_FAILED" OUTPUT_BINDING_FAILED = "OUTPUT_BINDING_FAILED" ATTEMPT_BUDGET_EXHAUSTED = "ATTEMPT_BUDGET_EXHAUSTED" + NODE_EXECUTION_INTERRUPTED = "NODE_EXECUTION_INTERRUPTED" @dataclass(frozen=True, slots=True) @@ -48,6 +51,7 @@ class NodeFailure: exception_type: str | None = None upstream_nodes: tuple[str, ...] = () output_name: str | None = None + output_port: str | None = None @dataclass(frozen=True, slots=True) @@ -59,6 +63,9 @@ class NodeResult: input: JsonValue = None value: JsonValue = None failure: NodeFailure | None = None + # JSON null is a valid bound input, so durable recovery needs a separate + # presence bit for outcomes that settled before input binding completed. + input_bound: bool = True @property def succeeded(self) -> bool: @@ -96,6 +103,10 @@ class NodeContext: completed: Mapping[str, JsonValue] attempt: int cancel_signal: CancellationSignal + run_id: str | None = None + attempt_id: str | None = None + idempotency_key: str | None = None + activity_key: str | None = None @dataclass(frozen=True, slots=True) @@ -118,6 +129,46 @@ def succeeded(self) -> bool: NodeHandler = Callable[[NodeContext], JsonValue | Awaitable[JsonValue]] +@dataclass(frozen=True, slots=True) +class _AttemptIdentity: + run_id: str + attempt_id: str + activity_key: str + + +class _SchedulerJournal(Protocol): + async def before_attempt( + self, + node: NodeSpec, + node_input: JsonValue, + attempt: int, + ) -> _AttemptIdentity: ... + + async def attempt_failed( + self, + failure: NodeFailure, + *, + will_retry: bool, + retry_delay_ms: float, + ) -> int: ... + + async def node_succeeded( + self, + node: NodeSpec, + node_input: JsonValue, + attempt: int, + output: JsonValue, + ) -> int: ... + + async def node_settled_without_attempt( + self, + node: NodeSpec, + result: NodeResult, + ) -> int: ... + + async def run_terminal(self, result: RunResult) -> None: ... + + class _BindingError(ValueError): pass @@ -134,6 +185,15 @@ class _HandlerCancelled(Exception): pass +def _consume_background_task_outcome(task: asyncio.Task[NodeResult]) -> None: + """Observe a detached durable task so a late failure is never unhandled.""" + + if task.cancelled(): + return + with suppress(asyncio.CancelledError): + _ = task.exception() + + def _snapshot_node_output(value: object) -> JsonValue: try: return portable_json_snapshot(value) @@ -143,6 +203,19 @@ def _snapshot_node_output(value: object) -> JsonValue: ) from exc +def _snapshot_context_graph(graph: GraphSpec) -> GraphSpec: + document = portable_json_snapshot( + graph.model_dump(mode="json", by_alias=True, exclude_unset=True) + ) + if type(document) is not dict: + raise AssertionError("graph context snapshot must be an object") + return GraphSpec.model_validate(document) + + +def _snapshot_compiled_graph(graph: CompiledGraph) -> CompiledGraph: + return compile_graph(_snapshot_context_graph(graph.spec)) + + async def _resolve_node_output(value: Awaitable[JsonValue]) -> JsonValue: """Snapshot an async handler result before its task yields completion.""" @@ -225,6 +298,7 @@ def _cancelled_result( *, status: NodeStatus, node_input: JsonValue = None, + input_bound: bool = True, ) -> NodeResult: return NodeResult( node_id=node_id, @@ -232,6 +306,7 @@ def _cancelled_result( status=status, attempts=attempts, input=node_input, + input_bound=input_bound, failure=NodeFailure( FailureCode.NODE_CANCELLED, f"node {node_id!r} was cancelled", @@ -267,9 +342,7 @@ def _effective_concurrency(self, graph: CompiledGraph) -> int: def _handler_for(self, node: NodeSpec) -> NodeHandler | None: handler = ( - self._handlers.get(node.id) - or self._handlers.get(node.kind) - or self._handlers.get("*") + self._handlers.get(node.id) or self._handlers.get(node.kind) or self._handlers.get("*") ) if handler is None and node.kind in {"transform", "barrier"}: return _identity_handler @@ -300,8 +373,17 @@ def _bind_input( def _retry_delay_ms(node: NodeSpec, failed_attempt: int) -> float: initial = float(node.retry.initial_delay_ms or 0) if node.retry else 0.0 multiplier = float(node.retry.backoff_multiplier or 1) if node.retry else 1.0 - maximum = float(node.retry.max_delay_ms or initial) if node.retry else initial - return min(maximum, initial * multiplier ** max(0, failed_attempt - 1)) + configured_maximum = ( + float(node.retry.max_delay_ms) + if node.retry and node.retry.max_delay_ms is not None + else initial + ) + maximum = max(initial, configured_maximum) + try: + scaled = initial * multiplier ** max(0, failed_attempt - 1) + except OverflowError: + scaled = float("inf") + return min(maximum, scaled) async def _attempt(self, handler: NodeHandler, context: NodeContext) -> JsonValue: if context.cancel_signal.cancelled: @@ -355,59 +437,114 @@ async def _execute_node( graph_input: JsonValue, completed: Mapping[str, JsonValue], cancel_signal: CancellationSignal, - claim_attempt: Callable[[], bool], + claim_attempt: Callable[[str], bool], + reserve_retry: Callable[[str], bool], + release_retry_reservation: Callable[[str], None], attempt_semaphore: asyncio.Semaphore, on_attempt_started: Callable[[], None], on_attempt_finished: Callable[[], None], + on_outcome_committed: Callable[[str, int], None], + *, + attempt_offset: int = 0, + journal: _SchedulerJournal | None = None, + initial_retry_delay_seconds: float = 0, ) -> NodeResult: node = graph.nodes[node_id] + attempts = attempt_offset + + async def settled_without_attempt(result: NodeResult) -> NodeResult: + if journal is not None: + ordinal = await journal.node_settled_without_attempt(node, result) + on_outcome_committed(node_id, ordinal) + release_retry_reservation(node_id) + return result + if cancel_signal.cancelled: - return _cancelled_result( - node_id, - sequence, - 0, - status=NodeStatus.FAILED, - node_input=node_input, + return await settled_without_attempt( + _cancelled_result( + node_id, + sequence, + attempts, + status=NodeStatus.FAILED, + node_input=node_input, + ) ) handler = self._handler_for(node) if handler is None: - return NodeResult( - node_id=node_id, - sequence=sequence, - status=NodeStatus.FAILED, - attempts=0, - input=node_input, - failure=NodeFailure( - FailureCode.EXECUTOR_NOT_FOUND, - f"no executor is registered for node {node_id!r} or kind {node.kind!r}", - node_id, - 0, - ), + return await settled_without_attempt( + NodeResult( + node_id=node_id, + sequence=sequence, + status=NodeStatus.FAILED, + attempts=attempts, + input=node_input, + failure=NodeFailure( + FailureCode.EXECUTOR_NOT_FOUND, + f"no executor is registered for node {node_id!r} or kind {node.kind!r}", + node_id, + attempts, + ), + ) ) max_attempts = (node.retry.max_attempts or 1) if node.retry else 1 - attempts = 0 last_failure: NodeFailure | None = None - while attempts < max_attempts: - if not await _acquire_attempt_slot(attempt_semaphore, cancel_signal): - return _cancelled_result( + if initial_retry_delay_seconds > 0 and not await _delay_or_cancel( + initial_retry_delay_seconds, + cancel_signal, + ): + return await settled_without_attempt( + _cancelled_result( node_id, sequence, attempts, status=NodeStatus.FAILED, node_input=node_input, ) - on_attempt_started() - try: - if cancel_signal.cancelled: - return _cancelled_result( + ) + if attempts >= max_attempts: + failure = NodeFailure( + FailureCode.ATTEMPT_BUDGET_EXHAUSTED, + f"node {node_id!r} has exhausted its durable attempt budget", + node_id, + attempts, + ) + return await settled_without_attempt( + NodeResult( + node_id=node_id, + sequence=sequence, + status=NodeStatus.FAILED, + attempts=attempts, + input=node_input, + failure=failure, + ) + ) + while attempts < max_attempts: + if not await _acquire_attempt_slot(attempt_semaphore, cancel_signal): + return await settled_without_attempt( + _cancelled_result( node_id, sequence, attempts, status=NodeStatus.FAILED, node_input=node_input, ) - if not claim_attempt(): + ) + attempt_counted = False + may_retry = False + delay_ms = 0.0 + try: + if cancel_signal.cancelled: + return await settled_without_attempt( + _cancelled_result( + node_id, + sequence, + attempts, + status=NodeStatus.FAILED, + node_input=node_input, + ) + ) + if not claim_attempt(node_id): if last_failure is None: last_failure = NodeFailure( FailureCode.ATTEMPT_BUDGET_EXHAUSTED, @@ -424,19 +561,30 @@ async def _execute_node( retryable=False, exception_type=last_failure.exception_type, ) - return NodeResult( - node_id=node_id, - sequence=sequence, - status=NodeStatus.FAILED, - attempts=attempts, - input=node_input, - failure=last_failure, + return await settled_without_attempt( + NodeResult( + node_id=node_id, + sequence=sequence, + status=NodeStatus.FAILED, + attempts=attempts, + input=node_input, + failure=last_failure, + ) ) attempts += 1 + identity = ( + await journal.before_attempt(node, node_input, attempts) + if journal is not None + else None + ) + on_attempt_started() + attempt_counted = True attempt_input = portable_json_snapshot(node_input) attempt_graph_input = portable_json_snapshot(graph_input) attempt_completed = portable_json_snapshot(dict(completed)) + context_graph = _snapshot_context_graph(graph.spec) + context_node = next(item for item in context_graph.nodes if item.id == node_id) if not isinstance(attempt_completed, dict): raise AssertionError("completed snapshot must be an object") mapping_input = ( @@ -445,14 +593,18 @@ async def _execute_node( else MappingProxyType({}) ) context = NodeContext( - graph=graph.spec, - node=node, + graph=context_graph, + node=context_node, input=attempt_input, graph_input=attempt_graph_input, inputs=mapping_input, completed=MappingProxyType(attempt_completed), attempt=attempts, cancel_signal=cancel_signal, + run_id=identity.run_id if identity is not None else None, + attempt_id=identity.attempt_id if identity is not None else None, + idempotency_key=(identity.activity_key if identity is not None else None), + activity_key=identity.activity_key if identity is not None else None, ) try: value = await self._attempt(handler, context) @@ -463,8 +615,7 @@ async def _execute_node( except _HandlerCancelled: code = FailureCode.NODE_EXECUTION_FAILED message = ( - f"node {node_id!r} failed: handler was cancelled without " - "run cancellation" + f"node {node_id!r} failed: handler was cancelled without run cancellation" ) exception_type = "CancelledError" except TimeoutError as exc: @@ -488,6 +639,14 @@ async def _execute_node( ) exception_type = type(exc).__name__ else: + if journal is not None: + ordinal = await journal.node_succeeded( + node, + node_input, + attempts, + value, + ) + on_outcome_committed(node_id, ordinal) return NodeResult( node_id=node_id, sequence=sequence, @@ -496,19 +655,36 @@ async def _execute_node( input=node_input, value=value, ) + + may_retry = ( + attempts < max_attempts + and code is not FailureCode.NODE_CANCELLED + and reserve_retry(node_id) + ) + last_failure = NodeFailure( + code, + message, + node_id, + attempts, + retryable=may_retry, + exception_type=exception_type, + ) + delay_ms = self._retry_delay_ms(node, attempts) if may_retry else 0.0 + if journal is not None: + ordinal = await journal.attempt_failed( + last_failure, + will_retry=may_retry, + retry_delay_ms=delay_ms, + ) + if not may_retry: + on_outcome_committed(node_id, ordinal) finally: - on_attempt_finished() + if attempt_counted: + on_attempt_finished() attempt_semaphore.release() - may_retry = attempts < max_attempts and code is not FailureCode.NODE_CANCELLED - last_failure = NodeFailure( - code, - message, - node_id, - attempts, - retryable=may_retry, - exception_type=exception_type, - ) + if last_failure is None: + raise AssertionError("failed node attempt omitted its failure") if not may_retry: return NodeResult( node_id=node_id, @@ -518,67 +694,118 @@ async def _execute_node( input=node_input, failure=last_failure, ) - delay_ms = self._retry_delay_ms(node, attempts) - if delay_ms > 0 and not await _delay_or_cancel( - delay_ms / 1000, cancel_signal - ): - return _cancelled_result( - node_id, - sequence, - attempts, - status=NodeStatus.FAILED, - node_input=node_input, + if delay_ms > 0 and not await _delay_or_cancel(delay_ms / 1000, cancel_signal): + return await settled_without_attempt( + _cancelled_result( + node_id, + sequence, + attempts, + status=NodeStatus.FAILED, + node_input=node_input, + ) ) raise AssertionError("bounded retry loop exited unexpectedly") - async def run( + async def _run( self, graph: CompiledGraph, graph_input: JsonValue, *, cancel_event: asyncio.Event | None = None, + journal: _SchedulerJournal | None = None, + initial_results: Mapping[str, NodeResult] | None = None, + attempt_offsets: Mapping[str, int] | None = None, + initial_total_attempts: int = 0, + initial_scheduled_order: tuple[str, ...] = (), + initial_completion_order: tuple[str, ...] = (), + initial_max_observed_concurrency: int = 0, + initial_retry_delays: Mapping[str, float] | None = None, ) -> RunResult: """Run a compiled graph and return deterministic, structured results.""" + graph = _snapshot_compiled_graph(graph) try: graph_input_snapshot = portable_json_snapshot(graph_input) except PortableJsonError: - raise TypeError( - "graph input must be a portable finite JSON value" - ) from None + raise TypeError("graph input must be a portable finite JSON value") from None if cancel_event is not None and not isinstance(cancel_event, asyncio.Event): raise TypeError("cancel_event must be an asyncio.Event") cancellation = CancellationSignal(cancel_event or asyncio.Event()) sequence = {node_id: index for index, node_id in enumerate(graph.topological_order)} + restored_results = dict(initial_results or {}) + if not set(restored_results).issubset(graph.nodes): + raise ValueError("initial_results contains a node outside the compiled graph") remaining = {node_id: len(graph.incoming[node_id]) for node_id in graph.nodes} + for restored_node_id in graph.topological_order: + if restored_node_id not in restored_results: + continue + for edge in graph.outgoing[restored_node_id]: + remaining[edge.target.node] -= 1 ready = sorted( - (node_id for node_id, count in remaining.items() if count == 0), + ( + node_id + for node_id, count in remaining.items() + if count == 0 and node_id not in restored_results + ), key=sequence.__getitem__, ) active: dict[asyncio.Task[NodeResult], str] = {} - results: dict[str, NodeResult] = {} - scheduled: list[str] = [] - completion: list[str] = [] + results: dict[str, NodeResult] = restored_results + scheduled: list[str] = list(initial_scheduled_order) + completion: list[str] = list(initial_completion_order) + durable_completion_prefix = tuple(initial_completion_order) + completion_ordinals: dict[str, int] = {} concurrency = self._effective_concurrency(graph) attempt_limit = ( graph.spec.policies.max_total_attempts if graph.spec.policies and graph.spec.policies.max_total_attempts is not None - else 2**63 - 1 + else MAX_SAFE_INTEGER ) - total_attempts = 0 + if ( + type(initial_total_attempts) is not int + or initial_total_attempts < 0 + or initial_total_attempts > MAX_SAFE_INTEGER + ): + raise TypeError("initial_total_attempts must be a non-negative safe integer") + total_attempts = initial_total_attempts + retry_reservations = set((initial_retry_delays or {}).keys()) + for node_id in retry_reservations: + if node_id not in graph.nodes: + raise ValueError(f"initial_retry_delays contains unknown node {node_id!r}") + if node_id in restored_results: + raise ValueError(f"initial retry reservation targets settled node {node_id!r}") + if total_attempts + len(retry_reservations) > attempt_limit: + raise ValueError( + "initial durable attempts and retry reservations exceed maxTotalAttempts" + ) running_attempts = 0 - max_observed_concurrency = 0 + max_observed_concurrency = initial_max_observed_concurrency attempt_semaphore = asyncio.Semaphore(concurrency) - def claim_attempt() -> bool: + def claim_attempt(node_id: str) -> bool: nonlocal total_attempts - if total_attempts >= attempt_limit: + if node_id in retry_reservations: + retry_reservations.remove(node_id) + total_attempts += 1 + return True + if total_attempts + len(retry_reservations) >= attempt_limit: return False total_attempts += 1 return True + def reserve_retry(node_id: str) -> bool: + if node_id in retry_reservations: + return False + if total_attempts + len(retry_reservations) >= attempt_limit: + return False + retry_reservations.add(node_id) + return True + + def release_retry_reservation(node_id: str) -> None: + retry_reservations.discard(node_id) + def on_attempt_started() -> None: nonlocal running_attempts, max_observed_concurrency running_attempts += 1 @@ -588,6 +815,11 @@ def on_attempt_finished() -> None: nonlocal running_attempts running_attempts -= 1 + def on_outcome_committed(node_id: str, ordinal: int) -> None: + if node_id in completion_ordinals: + raise RuntimeError(f"node {node_id!r} committed more than one terminal outcome") + completion_ordinals[node_id] = ordinal + def settle(node_id: str, result: NodeResult) -> None: results[node_id] = result completion.append(node_id) @@ -598,18 +830,28 @@ def settle(node_id: str, result: NodeResult) -> None: ready.append(target) ready.sort(key=sequence.__getitem__) - def launch_ready() -> None: + async def settle_without_attempt(node_id: str, result: NodeResult) -> None: + if journal is not None: + ordinal = await journal.node_settled_without_attempt(graph.nodes[node_id], result) + on_outcome_committed(node_id, ordinal) + release_retry_reservation(node_id) + settle(node_id, result) + + async def launch_ready() -> None: while ready: node_id = ready.pop(0) - scheduled.append(node_id) + attempt_offset = (attempt_offsets or {}).get(node_id, 0) + if node_id not in scheduled: + scheduled.append(node_id) if cancellation.cancelled: - settle( + await settle_without_attempt( node_id, _cancelled_result( node_id, sequence[node_id], - 0, + attempt_offset, status=NodeStatus.SKIPPED, + input_bound=False, ), ) continue @@ -623,18 +865,19 @@ def launch_ready() -> None: ) ) if upstream_failed: - settle( + await settle_without_attempt( node_id, NodeResult( node_id=node_id, sequence=sequence[node_id], status=NodeStatus.SKIPPED, - attempts=0, + attempts=attempt_offset, + input_bound=False, failure=NodeFailure( FailureCode.UPSTREAM_FAILED, "node did not start because upstream dependencies failed", node_id, - 0, + attempt_offset, upstream_nodes=upstream_failed, ), ), @@ -649,18 +892,19 @@ def launch_ready() -> None: results, ) except _BindingError as exc: - settle( + await settle_without_attempt( node_id, NodeResult( node_id=node_id, sequence=sequence[node_id], status=NodeStatus.FAILED, - attempts=0, + attempts=attempt_offset, + input_bound=False, failure=NodeFailure( FailureCode.INPUT_BINDING_FAILED, f"could not bind input for node {node_id!r}: {exc}", node_id, - 0, + attempt_offset, exception_type=type(exc).__name__, ), ), @@ -684,27 +928,46 @@ def launch_ready() -> None: completed, cancellation, claim_attempt, + reserve_retry, + release_retry_reservation, attempt_semaphore, on_attempt_started, on_attempt_finished, + on_outcome_committed, + attempt_offset=attempt_offset, + journal=journal, + initial_retry_delay_seconds=(initial_retry_delays or {}).get(node_id, 0), ) ) active[task] = node_id - while len(results) < len(graph.nodes): - launch_ready() - if not active: - break - done, _ = await asyncio.wait(active, return_when=asyncio.FIRST_COMPLETED) - for task in sorted(done, key=lambda item: sequence[active[item]]): - node_id = active.pop(task) - settle(node_id, task.result()) + try: + while len(results) < len(graph.nodes): + await launch_ready() + if not active: + break + done, _ = await asyncio.wait(active, return_when=asyncio.FIRST_COMPLETED) + for task in sorted(done, key=lambda item: sequence[active[item]]): + node_id = active.pop(task) + settle(node_id, task.result()) + except BaseException: + active_tasks = tuple(active) + if journal is not None: + # A journal failure is authoritative and must be surfaced even + # when an unrelated handler ignores cancellation. Observe each + # detached task's eventual outcome without delaying the error. + for task in active_tasks: + task.add_done_callback(_consume_background_task_outcome) + for task in active_tasks: + if not task.done(): + task.cancel() + if journal is None: + await asyncio.gather(*active_tasks, return_exceptions=True) + raise ordered_results = {node_id: results[node_id] for node_id in graph.topological_order} failures = [ - result.failure - for result in ordered_results.values() - if result.failure is not None + result.failure for result in ordered_results.values() if result.failure is not None ] output_values: dict[str, JsonValue] = {} outputs_complete = True @@ -728,6 +991,7 @@ def launch_ready() -> None: 0, exception_type=type(exc).__name__, output_name=output_name, + output_port=endpoint.port, ) ) @@ -738,17 +1002,44 @@ def launch_ready() -> None: else: status = RunStatus.FAILED outputs = MappingProxyType(output_values) if outputs_complete else None - return RunResult( + completion_order = tuple(completion) + if journal is not None: + committed_suffix = tuple( + node_id + for node_id, _ in sorted( + completion_ordinals.items(), + key=lambda item: item[1], + ) + if node_id not in durable_completion_prefix + ) + completion_order = durable_completion_prefix + committed_suffix + if len(completion_order) != len(results) or set(completion_order) != set(results): + raise RuntimeError("durable run is missing an explicit committed node outcome") + run_result = RunResult( status=status, graph_hash=graph.graph_hash, nodes=MappingProxyType(ordered_results), outputs=outputs, failures=tuple(failures), scheduled_order=tuple(scheduled), - completion_order=tuple(completion), + completion_order=completion_order, max_observed_concurrency=max_observed_concurrency, total_attempts=total_attempts, ) + if journal is not None: + await journal.run_terminal(run_result) + return run_result + + async def run( + self, + graph: CompiledGraph, + graph_input: JsonValue, + *, + cancel_event: asyncio.Event | None = None, + ) -> RunResult: + """Run a compiled graph and return deterministic, structured results.""" + + return await self._run(graph, graph_input, cancel_event=cancel_event) async def run_graph( diff --git a/python/tests/test_compiler.py b/python/tests/test_compiler.py index d832239..b47009e 100644 --- a/python/tests/test_compiler.py +++ b/python/tests/test_compiler.py @@ -30,6 +30,7 @@ def test_diamond_topological_layers_match_conformance() -> None: ("invalid-missing-endpoint.graph.json", DiagnosticCode.MISSING_TARGET), ("invalid-cycle.graph.json", DiagnosticCode.CYCLE), ("invalid-unreachable.graph.json", DiagnosticCode.UNREACHABLE_NODE), + ("invalid-oversized-timers.graph.json", DiagnosticCode.INVALID_GRAPH), ], ) def test_invalid_conformance_fixtures(fixture: str, code: DiagnosticCode) -> None: @@ -143,3 +144,85 @@ def test_surrogate_pair_key_collision_returns_invalid_graph() -> None: assert result.graph is None assert [item.code for item in result.diagnostics] == [DiagnosticCode.INVALID_GRAPH] assert result.diagnostics[0].message == "graph input could not be inspected safely" + + +@pytest.mark.parametrize( + "policy_name", + [ + "maxConcurrency", + "maxDynamicNodes", + "maxDepth", + "maxFanOut", + "maxTotalAttempts", + "maxDurationMs", + ], +) +def test_graph_policy_integers_reject_values_above_javascript_safe_range( + policy_name: str, +) -> None: + document = load_fixture("diamond.graph.json") + document["policies"] = {policy_name: 2**53} + + result = try_compile_graph(document) + + assert result.graph is None + assert [item.code for item in result.diagnostics] == [DiagnosticCode.INVALID_GRAPH] + + +@pytest.mark.parametrize( + ("field_name", "value"), + [ + ("timeoutMs", 2**53), + ("initialDelayMs", 2**53), + ("maxDelayMs", 2**53), + ("backoffMultiplier", float(2**53)), + ], +) +def test_node_timing_numbers_must_be_portable(field_name: str, value: object) -> None: + document = load_fixture("diamond.graph.json") + node = document["nodes"][0] # type: ignore[index] + if field_name == "timeoutMs": + node[field_name] = value + else: + node["retry"] = {field_name: value} + + result = try_compile_graph(document) + + assert result.graph is None + assert [item.code for item in result.diagnostics] == [DiagnosticCode.INVALID_GRAPH] + + +def test_timer_fields_accept_the_shared_32_bit_maximum() -> None: + document = load_fixture("diamond.graph.json") + node = document["nodes"][0] # type: ignore[index] + node["timeoutMs"] = 2**31 - 1 + node["retry"] = { + "initialDelayMs": 2**31 - 1, + "maxDelayMs": 2**31 - 1, + } + document["policies"] = {"maxDurationMs": 2**31 - 1} + + assert try_compile_graph(document).valid + + +@pytest.mark.parametrize("value", [float("nan"), float("inf"), float(2**53)]) +def test_max_cost_must_be_finite_portable_json(value: float) -> None: + document = load_fixture("diamond.graph.json") + document["policies"] = {"maxCostUsd": value} + + result = try_compile_graph(document) + + assert result.graph is None + assert [item.code for item in result.diagnostics] == [DiagnosticCode.INVALID_GRAPH] + + +def test_compiling_a_model_instance_detaches_and_revalidates_nested_json() -> None: + compiled = compile_graph(load_fixture("diamond.graph.json")) + config = compiled.spec.nodes[0].config + assert isinstance(config, dict) + config["unsafe"] = 2**53 + + result = try_compile_graph(compiled.spec) + + assert result.graph is None + assert [item.code for item in result.diagnostics] == [DiagnosticCode.INVALID_GRAPH] diff --git a/python/tests/test_durable_json.py b/python/tests/test_durable_json.py new file mode 100644 index 0000000..3b706ed --- /dev/null +++ b/python/tests/test_durable_json.py @@ -0,0 +1,102 @@ +from __future__ import annotations + +import json +import math +import struct +from pathlib import Path + +import pytest + +from graph_engineering import canonical_json +from graph_engineering.durable_json import ( + DurableJsonError, + decode_durable_json, + durable_json_hash, + encode_durable_json, +) + +ROOT = Path(__file__).parents[2] + + +def test_tagged_durable_json_round_trips_every_runtime_value_kind() -> None: + value = { + "array": [None, True, "text", 42, 1.5], + "object": {"z": 0.1, "é": -2}, + } + + encoded = encode_durable_json(value) + + assert encoded == [ + "o", + [ + [ + "array", + [ + "a", + [["n"], ["b", True], ["s", "text"], ["i", 42], ["f", "3ff8000000000000"]], + ], + ], + [ + "object", + ["o", [["z", ["f", "3fb999999999999a"]], ["é", ["i", -2]]]], + ], + ], + ] + assert decode_durable_json(encoded) == value + assert durable_json_hash(value) == durable_json_hash(decode_durable_json(encoded)) + + +def test_integer_valued_floats_and_negative_zero_normalize_to_integer_tags() -> None: + assert encode_durable_json([1.0, -0.0]) == ["a", [["i", 1], ["i", 0]]] + + +@pytest.mark.parametrize("value", [math.nan, math.inf, -math.inf, 2**53]) +def test_encoder_rejects_values_outside_the_portable_runtime_boundary(value: object) -> None: + with pytest.raises(DurableJsonError): + encode_durable_json(value) + + +@pytest.mark.parametrize( + "encoded", + [ + [], + ["unknown"], + ["n", None], + ["i", True], + ["i", 2**53], + ["f", "3ff0000000000000"], + ["f", "7ff0000000000000"], + ["f", "3FF8000000000000"], + ["o", [["b", ["n"]], ["a", ["n"]]]], + ["o", [["a", ["n"]], ["a", ["n"]]]], + ], +) +def test_decoder_rejects_noncanonical_or_malformed_tagged_values(encoded: object) -> None: + with pytest.raises(DurableJsonError): + decode_durable_json(encoded) + + +def test_shared_durable_json_conformance_corpus() -> None: + corpus = json.loads( + (ROOT / "spec/conformance/durable-json.case.json").read_text(encoding="utf-8") + ) + for case in corpus["validCases"]: + source = case["source"] + value = ( + source["value"] + if source["kind"] == "json" + else struct.unpack(">d", bytes.fromhex(source["bits"]))[0] + ) + encoded = encode_durable_json(value) + assert encoded == case["expect"]["encoding"], case["name"] + assert canonical_json(encoded) == case["expect"]["canonicalJson"], case["name"] + assert durable_json_hash(value) == case["expect"]["sha256"], case["name"] + decoded = decode_durable_json(encoded) + if isinstance(value, float) and value == 0: + assert decoded == 0 + else: + assert decoded == value + + for case in corpus["invalidEncodedCases"]: + with pytest.raises(DurableJsonError): + decode_durable_json(case["encoding"]) diff --git a/python/tests/test_durable_scheduler.py b/python/tests/test_durable_scheduler.py new file mode 100644 index 0000000..aa690ab --- /dev/null +++ b/python/tests/test_durable_scheduler.py @@ -0,0 +1,2856 @@ +from __future__ import annotations + +import asyncio +import json +from collections.abc import Sequence +from pathlib import Path +from typing import Any + +import pytest + +from graph_engineering import ( + AsyncScheduler, + DurableRunError, + DurableRunErrorCode, + FailureCode, + GraphCompileError, + GraphEvent, + NodeContext, + NodeFailure, + RunStatus, + compile_graph, + decode_durable_json, + durable_json_hash, + encode_durable_json, + resume_graph_run, + start_graph_run, +) +from graph_engineering.canonical import canonical_sha256 +from graph_engineering.durable import _DurableJournal, _EventDraft, _strict_rfc3339 +from graph_engineering.persistence import EventStore, MemoryEventStore, VersionConflictError +from graph_engineering.scheduler import _AttemptIdentity + +ROOT = Path(__file__).parents[2] +FIXED_TIME = "2026-07-26T12:00:00.000Z" + + +class ProcessLost(BaseException): + pass + + +def fixed_clock() -> str: + return FIXED_TIME + + +def test_durable_timestamp_parser_consumes_shared_strict_rfc3339_corpus() -> None: + corpus = json.loads((ROOT / "spec/conformance/strict-rfc3339.case.json").read_text()) + + for timestamp in corpus["valid"]: + _strict_rfc3339(timestamp) + for timestamp in corpus["invalid"]: + with pytest.raises(ValueError): + _strict_rfc3339(timestamp) + + +def graph( + node: dict[str, Any] | None = None, + *, + policies: dict[str, Any] | None = None, +) -> Any: + document = { + "apiVersion": "graphengineering.reacher-z.github.io/v1alpha1", + "kind": "Graph", + "metadata": {"name": "durable-test", "version": "1"}, + "inputSchema": {}, + "outputSchema": {}, + "entrypoints": ["root"], + "outputs": {"result": {"node": "root"}}, + "nodes": [ + node + or { + "id": "root", + "kind": "agent", + "inputSchema": {}, + "outputSchema": {}, + "config": {}, + "sideEffects": "none", + } + ], + "edges": [], + } + if policies is not None: + document["policies"] = policies + return compile_graph(document) + + +async def events(store: EventStore, run_id: str) -> tuple[GraphEvent, ...]: + return await store.read(run_id) + + +def resign(event: GraphEvent, data: dict[str, Any]) -> GraphEvent: + return event.model_copy(update={"data": data, "payload_hash": canonical_sha256(data)}) + + +def forged_event( + *, + run_id: str, + sequence: int, + event_type: str, + data: dict[str, Any], + node_id: str | None = None, + attempt: int | None = None, +) -> GraphEvent: + document: dict[str, Any] = { + "apiVersion": "graphengineering.reacher-z.github.io/events/v1alpha1", + "eventId": f"forged:{sequence}", + "type": event_type, + "timestamp": FIXED_TIME, + "runId": run_id, + "graphRevision": 1, + "sequence": sequence, + "payloadHash": canonical_sha256(data), + "redacted": True, + "data": data, + } + if node_id is not None: + document["nodeId"] = node_id + if attempt is not None: + document["attempt"] = attempt + return GraphEvent.model_validate(document) + + +class TrackingStore: + def __init__(self, delegate: MemoryEventStore | None = None) -> None: + self.delegate = delegate or MemoryEventStore() + self.read_calls = 0 + self.append_calls = 0 + + async def read(self, run_id: str, from_sequence: int = 0) -> tuple[GraphEvent, ...]: + self.read_calls += 1 + return await self.delegate.read(run_id, from_sequence) + + async def append( + self, + run_id: str, + expected_version: int, + values: Sequence[GraphEvent], + ) -> int: + self.append_calls += 1 + return await self.delegate.append(run_id, expected_version, values) + + +def test_start_commits_claim_before_handler_and_terminal_resume_is_idempotent() -> None: + async def scenario() -> None: + store = MemoryEventStore() + observed_identity: tuple[str | None, str | None, str | None] | None = None + + async def handler(context: NodeContext) -> Any: + nonlocal observed_identity + history = await store.read("run-start") + assert history[-1].type == "NodeStarted" + observed_identity = ( + context.run_id, + context.attempt_id, + context.idempotency_key, + ) + return {"decimal": 0.1} + + result = await start_graph_run( + graph(), + {"seed": 1.5}, + {"root": handler}, + run_id="run-start", + implementation_id="handlers@1", + event_store=store, + clock=fixed_clock, + ) + history = await store.read("run-start") + assert result.status is RunStatus.SUCCEEDED + assert [item.type for item in history] == [ + "RunCreated", + "RunStarted", + "NodeScheduled", + "NodeStarted", + "NodeSucceeded", + "RunSucceeded", + ] + assert all(item.payload_hash == canonical_sha256(item.data) for item in history) + assert observed_identity is not None + assert observed_identity[0] == "run-start" + assert observed_identity[1] == "run-start/root/1" + assert observed_identity[2] == history[2].data["activityKey"] + + calls = 0 + + def must_not_run(_: NodeContext) -> Any: + nonlocal calls + calls += 1 + raise AssertionError("terminal resume invoked an executor") + + before = len(history) + resumed = await resume_graph_run( + graph(), + {"root": must_not_run}, + run_id="run-start", + implementation_id="handlers@1", + event_store=store, + clock=fixed_clock, + ) + assert resumed == result + assert calls == 0 + assert len(await store.read("run-start")) == before + + with pytest.raises(DurableRunError) as duplicate: + await start_graph_run( + graph(), + {}, + run_id="run-start", + implementation_id="handlers@1", + event_store=store, + clock=fixed_clock, + ) + assert duplicate.value.code is DurableRunErrorCode.RUN_ALREADY_EXISTS + + asyncio.run(scenario()) + + +def test_shared_resume_fixture_reuses_success_and_advances_open_attempt() -> None: + fixture = json.loads( + (ROOT / "spec/conformance/durable-resume.case.json").read_text(encoding="utf-8") + ) + compiled = compile_graph(fixture["graph"]) + + async def scenario() -> None: + store = MemoryEventStore() + + async def left(_: NodeContext) -> Any: + return fixture["preCrash"]["succeeded"]["output"] + + async def right_crashes(_: NodeContext) -> Any: + await asyncio.sleep(0.02) + raise ProcessLost + + with pytest.raises(ProcessLost): + await start_graph_run( + compiled, + fixture["graphInput"], + {"left": left, "right": right_crashes, "merge": lambda _: None}, + run_id=fixture["runId"], + implementation_id=fixture["implementationId"], + event_store=store, + clock=fixed_clock, + ) + pre_resume_count = len(await store.read(fixture["runId"])) + + calls: list[tuple[str, int]] = [] + merge_input: object = None + right_keys: list[str | None] = [] + + def never_left(_: NodeContext) -> Any: + raise AssertionError("committed successful node ran again") + + def right(context: NodeContext) -> Any: + calls.append((context.node.id, context.attempt)) + right_keys.append(context.activity_key) + return fixture["resumeReturns"]["right"] + + def merge(context: NodeContext) -> Any: + nonlocal merge_input + calls.append((context.node.id, context.attempt)) + merge_input = context.input + return fixture["resumeReturns"]["merge"] + + result = await resume_graph_run( + compiled, + {"left": never_left, "right": right, "merge": merge}, + run_id=fixture["runId"], + implementation_id=fixture["implementationId"], + event_store=store, + clock=fixed_clock, + ) + assert calls == [ + (item["nodeId"], item["attempt"]) for item in fixture["expect"]["executorCalls"] + ] + assert merge_input == fixture["expect"]["mergeInput"] + assert result.status.value == fixture["expect"]["status"] + assert dict(result.outputs or {}) == fixture["expect"]["output"] + assert result.total_attempts == fixture["expect"]["totalAttempts"] + + resumed_history = await store.read(fixture["runId"]) + assert [item.type for item in resumed_history[pre_resume_count : pre_resume_count + 3]] == [ + "RunResumed", + "NodeAttemptFailed", + "NodeRetried", + ] + right_schedules = [ + item + for item in resumed_history + if item.type == "NodeScheduled" and item.node_id == "right" + ] + assert len(right_schedules) == 2 + assert right_schedules[0].data["activityKey"] == right_schedules[1].data["activityKey"] + assert right_keys == [right_schedules[0].data["activityKey"]] + + terminal_version = len(await store.read(fixture["runId"])) + terminal = await resume_graph_run( + compiled, + {"*": never_left}, + run_id=fixture["runId"], + implementation_id=fixture["implementationId"], + event_store=store, + clock=fixed_clock, + ) + assert terminal == result + assert len(await store.read(fixture["runId"])) == terminal_version + + asyncio.run(scenario()) + + +def test_resume_rejects_implementation_graph_input_and_payload_identity() -> None: + async def scenario() -> None: + store = MemoryEventStore() + await start_graph_run( + graph(), + {"seed": 1}, + {"root": lambda _: "ok"}, + run_id="identity", + implementation_id="v1", + event_store=store, + clock=fixed_clock, + ) + with pytest.raises(DurableRunError) as implementation: + await resume_graph_run( + graph(), + run_id="identity", + implementation_id="v2", + event_store=store, + ) + assert implementation.value.code is DurableRunErrorCode.IMPLEMENTATION_MISMATCH + + changed = graph( + { + "id": "root", + "kind": "agent", + "inputSchema": {}, + "outputSchema": {}, + "config": {"changed": True}, + "sideEffects": "none", + } + ) + with pytest.raises(DurableRunError) as graph_error: + await resume_graph_run( + changed, + run_id="identity", + implementation_id="v1", + event_store=store, + ) + assert graph_error.value.code is DurableRunErrorCode.GRAPH_HASH_MISMATCH + + original = await store.read("identity") + tampered_store = MemoryEventStore() + data = dict(original[0].data) + data["inputHash"] = "0" * 64 + tampered = original[0].model_copy( + update={"data": data, "payload_hash": canonical_sha256(data)} + ) + await tampered_store.append("identity", -1, (tampered, *original[1:])) + with pytest.raises(DurableRunError) as input_error: + await resume_graph_run( + graph(), + run_id="identity", + implementation_id="v1", + event_store=tampered_store, + ) + assert input_error.value.code is DurableRunErrorCode.INPUT_HASH_MISMATCH + + bad_hash_store = MemoryEventStore() + bad_hash = original[2].model_copy(update={"payload_hash": "f" * 64}) + await bad_hash_store.append( + "identity", + -1, + (*original[:2], bad_hash, *original[3:]), + ) + with pytest.raises(DurableRunError) as history_error: + await resume_graph_run( + graph(), + run_id="identity", + implementation_id="v1", + event_store=bad_hash_store, + ) + assert history_error.value.code is DurableRunErrorCode.INVALID_RUN_HISTORY + + asyncio.run(scenario()) + + +def test_undeclared_interrupted_side_effect_is_in_doubt_and_never_reinvoked() -> None: + unsafe_graph = graph( + { + "id": "root", + "kind": "agent", + "inputSchema": {}, + "outputSchema": {}, + "config": {}, + "retry": {"maxAttempts": 2}, + } + ) + + async def scenario() -> None: + store = MemoryEventStore() + with pytest.raises(ProcessLost): + await start_graph_run( + unsafe_graph, + {}, + {"root": lambda _: (_ for _ in ()).throw(ProcessLost())}, + run_id="unsafe", + implementation_id="v1", + event_store=store, + clock=fixed_clock, + ) + calls = 0 + + def handler(_: NodeContext) -> Any: + nonlocal calls + calls += 1 + return "bad" + + with pytest.raises(DurableRunError) as error: + await resume_graph_run( + unsafe_graph, + {"root": handler}, + run_id="unsafe", + implementation_id="v1", + event_store=store, + clock=fixed_clock, + ) + assert error.value.code is DurableRunErrorCode.IN_DOUBT_SIDE_EFFECT + assert calls == 0 + event_count = len(await store.read("unsafe")) + with pytest.raises(DurableRunError) as repeated: + await resume_graph_run( + unsafe_graph, + {"root": handler}, + run_id="unsafe", + implementation_id="v1", + event_store=store, + clock=fixed_clock, + ) + assert repeated.value.code is DurableRunErrorCode.IN_DOUBT_SIDE_EFFECT + assert calls == 0 + assert len(await store.read("unsafe")) == event_count + + asyncio.run(scenario()) + + +def test_interrupted_idempotent_node_reuses_activity_key_on_retry() -> None: + idempotent_graph = graph( + { + "id": "root", + "kind": "agent", + "inputSchema": {}, + "outputSchema": {}, + "config": {}, + "retry": {"maxAttempts": 2}, + "sideEffects": "idempotent", + } + ) + + async def scenario() -> None: + store = MemoryEventStore() + first_key: str | None = None + + def crashes(context: NodeContext) -> Any: + nonlocal first_key + first_key = context.activity_key + raise ProcessLost + + with pytest.raises(ProcessLost): + await start_graph_run( + idempotent_graph, + {"value": 0.1}, + {"root": crashes}, + run_id="idempotent", + implementation_id="v1", + event_store=store, + clock=fixed_clock, + ) + second_key: str | None = None + + def succeeds(context: NodeContext) -> Any: + nonlocal second_key + second_key = context.idempotency_key + assert context.attempt == 2 + return "ok" + + result = await resume_graph_run( + idempotent_graph, + {"root": succeeds}, + run_id="idempotent", + implementation_id="v1", + event_store=store, + clock=fixed_clock, + ) + assert result.succeeded + assert first_key == second_key + + asyncio.run(scenario()) + + +def test_resigned_forged_terminal_output_is_invalid_history() -> None: + async def scenario() -> None: + source = MemoryEventStore() + await start_graph_run( + graph(), + {"seed": 1}, + {"root": lambda _: "real"}, + run_id="forged-terminal", + implementation_id="v1", + event_store=source, + clock=fixed_clock, + ) + history = list(await source.read("forged-terminal")) + terminal = history[-1] + result = decode_durable_json(terminal.data["result"]) + assert isinstance(result, dict) + result["output"] = {"result": "forged"} + terminal_data = {"result": encode_durable_json(result)} + history[-1] = resign(terminal, terminal_data) + forged = MemoryEventStore() + await forged.append("forged-terminal", -1, history) + + with pytest.raises(DurableRunError) as error: + await resume_graph_run( + graph(), + run_id="forged-terminal", + implementation_id="v1", + event_store=forged, + ) + assert error.value.code is DurableRunErrorCode.INVALID_RUN_HISTORY + + asyncio.run(scenario()) + + +@pytest.mark.parametrize( + "field_name", + ["status", "nodes", "failures", "scheduledOrder", "completionOrder", "maxConcurrency"], +) +def test_resigned_forged_terminal_projection_is_invalid_history(field_name: str) -> None: + async def scenario() -> None: + run_id = f"forged-terminal-{field_name}" + source = MemoryEventStore() + await start_graph_run( + graph(), + {}, + {"root": lambda _: "real"}, + run_id=run_id, + implementation_id="v1", + event_store=source, + clock=fixed_clock, + ) + history = list(await source.read(run_id)) + terminal = history[-1] + result = decode_durable_json(terminal.data["result"]) + assert isinstance(result, dict) + if field_name == "status": + result["status"] = "failed" + elif field_name == "nodes": + nodes = result["nodes"] + assert isinstance(nodes, list) and isinstance(nodes[0], dict) + nodes[0]["output"] = "forged" + elif field_name == "failures": + failures = result["failures"] + assert isinstance(failures, list) + failures.append( + { + "phase": "execute", + "code": "NODE_EXECUTION_FAILED", + "message": "forged", + "nodeId": "root", + "attempt": 1, + "retryable": False, + } + ) + elif field_name == "scheduledOrder": + result["scheduledOrder"] = [] + elif field_name == "completionOrder": + result["completionOrder"] = [] + else: + result["maxObservedConcurrency"] = 2 + history[-1] = resign(terminal, {"result": encode_durable_json(result)}) + forged = MemoryEventStore() + await forged.append(run_id, -1, history) + + with pytest.raises(DurableRunError) as error: + await resume_graph_run( + graph(), + run_id=run_id, + implementation_id="v1", + event_store=forged, + ) + assert error.value.code is DurableRunErrorCode.INVALID_RUN_HISTORY + + asyncio.run(scenario()) + + +def test_self_consistent_forged_scheduled_input_is_invalid_history() -> None: + async def scenario() -> None: + source = MemoryEventStore() + await start_graph_run( + graph(), + {"seed": 1}, + {"root": lambda _: "ok"}, + run_id="forged-input", + implementation_id="v1", + event_store=source, + clock=fixed_clock, + ) + history = list(await source.read("forged-input")) + scheduled_index = next( + index for index, item in enumerate(history) if item.type == "NodeScheduled" + ) + started_index = next( + index for index, item in enumerate(history) if item.type == "NodeStarted" + ) + succeeded_index = next( + index for index, item in enumerate(history) if item.type == "NodeSucceeded" + ) + forged_input = {"seed": 999} + input_hash = durable_json_hash(forged_input) + activity_key = durable_json_hash( + ["activity/v1alpha1", "forged-input", 1, "root", input_hash] + ) + history[scheduled_index] = resign( + history[scheduled_index], + { + "input": encode_durable_json(forged_input), + "inputHash": input_hash, + "activityKey": activity_key, + "sideEffects": "none", + }, + ) + history[started_index] = resign( + history[started_index], + {"inputHash": input_hash, "activityKey": activity_key}, + ) + succeeded_data = dict(history[succeeded_index].data) + succeeded_data["inputHash"] = input_hash + history[succeeded_index] = resign(history[succeeded_index], succeeded_data) + terminal = history[-1] + result = decode_durable_json(terminal.data["result"]) + assert isinstance(result, dict) + nodes = result["nodes"] + assert isinstance(nodes, list) and isinstance(nodes[0], dict) + nodes[0]["input"] = forged_input + history[-1] = resign(terminal, {"result": encode_durable_json(result)}) + forged = MemoryEventStore() + await forged.append("forged-input", -1, history) + + with pytest.raises(DurableRunError) as error: + await resume_graph_run( + graph(), + run_id="forged-input", + implementation_id="v1", + event_store=forged, + ) + assert error.value.code is DurableRunErrorCode.INVALID_RUN_HISTORY + + asyncio.run(scenario()) + + +def test_run_resumed_lists_must_match_folded_facts_without_overlap() -> None: + retry_graph = graph( + { + "id": "root", + "kind": "agent", + "inputSchema": {}, + "outputSchema": {}, + "config": {}, + "retry": {"maxAttempts": 2}, + "sideEffects": "none", + } + ) + + async def scenario() -> None: + source = MemoryEventStore() + with pytest.raises(ProcessLost): + await start_graph_run( + retry_graph, + {}, + {"root": lambda _: (_ for _ in ()).throw(ProcessLost())}, + run_id="forged-resume-lists", + implementation_id="v1", + event_store=source, + clock=fixed_clock, + ) + await resume_graph_run( + retry_graph, + {"root": lambda _: "ok"}, + run_id="forged-resume-lists", + implementation_id="v1", + event_store=source, + clock=fixed_clock, + ) + history = list(await source.read("forged-resume-lists")) + index = next( + item_index for item_index, item in enumerate(history) if item.type == "RunResumed" + ) + history[index] = resign( + history[index], + {"reusedNodeIds": ["root"], "interruptedNodeIds": ["root"]}, + ) + forged = MemoryEventStore() + await forged.append("forged-resume-lists", -1, history) + + with pytest.raises(DurableRunError) as error: + await resume_graph_run( + retry_graph, + run_id="forged-resume-lists", + implementation_id="v1", + event_store=forged, + ) + assert error.value.code is DurableRunErrorCode.INVALID_RUN_HISTORY + + asyncio.run(scenario()) + + +@pytest.mark.parametrize( + ("max_attempts", "max_total_attempts"), + [(1, None), (2, 1)], +) +def test_retry_beyond_node_or_global_attempt_budget_is_invalid_history( + max_attempts: int, + max_total_attempts: int | None, +) -> None: + limited_graph = graph( + { + "id": "root", + "kind": "agent", + "inputSchema": {}, + "outputSchema": {}, + "config": {}, + "retry": {"maxAttempts": max_attempts}, + "sideEffects": "none", + }, + policies=( + {"maxTotalAttempts": max_total_attempts} if max_total_attempts is not None else None + ), + ) + run_id = f"over-budget-{max_attempts}-{max_total_attempts}" + + async def scenario() -> None: + store = MemoryEventStore() + with pytest.raises(ProcessLost): + await start_graph_run( + limited_graph, + {}, + {"root": lambda _: (_ for _ in ()).throw(ProcessLost())}, + run_id=run_id, + implementation_id="v1", + event_store=store, + clock=fixed_clock, + ) + history = await store.read(run_id) + version = history[-1].sequence + scheduled = next(item for item in history if item.type == "NodeScheduled") + activity_key = str(scheduled.data["activityKey"]) + input_hash = str(scheduled.data["inputHash"]) + failure = { + "phase": "execute", + "code": "NODE_EXECUTION_INTERRUPTED", + "message": "process ended before the attempt outcome was durably recorded", + "nodeId": "root", + "attempt": 1, + "retryable": True, + "causeName": "ProcessLost", + } + suffix = ( + forged_event( + run_id=run_id, + sequence=version + 1, + event_type="NodeAttemptFailed", + data={"terminal": False, "failure": failure}, + node_id="root", + attempt=1, + ), + forged_event( + run_id=run_id, + sequence=version + 2, + event_type="NodeRetried", + data={"availableAt": FIXED_TIME, "activityKey": activity_key}, + node_id="root", + attempt=2, + ), + forged_event( + run_id=run_id, + sequence=version + 3, + event_type="NodeScheduled", + data={ + "input": scheduled.data["input"], + "inputHash": input_hash, + "activityKey": activity_key, + "sideEffects": "none", + }, + node_id="root", + attempt=2, + ), + forged_event( + run_id=run_id, + sequence=version + 4, + event_type="NodeStarted", + data={"inputHash": input_hash, "activityKey": activity_key}, + node_id="root", + attempt=2, + ), + ) + await store.append(run_id, version, suffix) + + with pytest.raises(DurableRunError) as error: + await resume_graph_run( + limited_graph, + run_id=run_id, + implementation_id="v1", + event_store=store, + ) + assert error.value.code is DurableRunErrorCode.INVALID_RUN_HISTORY + + asyncio.run(scenario()) + + +class ConflictStore: + def __init__(self, delegate: MemoryEventStore) -> None: + self.delegate = delegate + + async def read(self, run_id: str, from_sequence: int = 0) -> tuple[GraphEvent, ...]: + return await self.delegate.read(run_id, from_sequence) + + async def append( + self, + run_id: str, + expected_version: int, + values: Sequence[GraphEvent], + ) -> int: + actual = len(await self.delegate.read(run_id)) - 1 + raise VersionConflictError(run_id, expected_version, actual + 1) + + +def test_resume_cas_conflict_invokes_no_executor() -> None: + async def scenario() -> None: + store = MemoryEventStore() + with pytest.raises(ProcessLost): + await start_graph_run( + graph( + { + "id": "root", + "kind": "agent", + "inputSchema": {}, + "outputSchema": {}, + "config": {}, + "retry": {"maxAttempts": 2}, + "sideEffects": "none", + } + ), + {}, + {"root": lambda _: (_ for _ in ()).throw(ProcessLost())}, + run_id="conflict", + implementation_id="v1", + event_store=store, + clock=fixed_clock, + ) + calls = 0 + before = len(await store.read("conflict")) + + def handler(_: NodeContext) -> Any: + nonlocal calls + calls += 1 + return "bad" + + with pytest.raises(DurableRunError) as error: + await resume_graph_run( + graph( + { + "id": "root", + "kind": "agent", + "inputSchema": {}, + "outputSchema": {}, + "config": {}, + "retry": {"maxAttempts": 2}, + "sideEffects": "none", + } + ), + {"root": handler}, + run_id="conflict", + implementation_id="v1", + event_store=ConflictStore(store), + clock=fixed_clock, + ) + assert error.value.code is DurableRunErrorCode.RESUME_CONFLICT + assert calls == 0 + assert len(await store.read("conflict")) == before + + asyncio.run(scenario()) + + +def test_durable_entrypoints_recompile_a_detached_graph_before_store_access() -> None: + async def scenario() -> None: + compiled = graph( + { + "id": "root", + "kind": "agent", + "inputSchema": {}, + "outputSchema": {}, + "config": {}, + "sideEffects": "none", + } + ) + config = compiled.spec.nodes[0].config + assert isinstance(config, dict) + config["unsafe"] = 2**53 + store = TrackingStore() + + with pytest.raises(GraphCompileError): + await start_graph_run( + compiled, + {}, + run_id="unsafe-start", + implementation_id="v1", + event_store=store, + ) + with pytest.raises(GraphCompileError): + await resume_graph_run( + compiled, + run_id="unsafe-resume", + implementation_id="v1", + event_store=store, + ) + + assert store.read_calls == 0 + assert store.append_calls == 0 + + asyncio.run(scenario()) + + +def test_mutating_the_original_compiled_graph_does_not_reach_later_handlers() -> None: + compiled = compile_graph( + { + "apiVersion": "graphengineering.reacher-z.github.io/v1alpha1", + "kind": "Graph", + "metadata": {"name": "snapshot-isolation", "version": "1"}, + "inputSchema": {}, + "outputSchema": {}, + "entrypoints": ["root"], + "outputs": {"result": {"node": "next"}}, + "nodes": [ + { + "id": "root", + "kind": "agent", + "inputSchema": {}, + "outputSchema": {}, + "config": {}, + "sideEffects": "none", + }, + { + "id": "next", + "kind": "agent", + "inputSchema": {}, + "outputSchema": {}, + "config": {"marker": "original"}, + "sideEffects": "none", + }, + ], + "edges": [{"id": "root-next", "from": {"node": "root"}, "to": {"node": "next"}}], + } + ) + + async def scenario() -> None: + store = MemoryEventStore() + + def mutate_original(_: NodeContext) -> Any: + original_config = compiled.spec.nodes[1].config + assert isinstance(original_config, dict) + original_config["marker"] = "mutated" + return "root" + + def observe_fresh(context: NodeContext) -> Any: + assert isinstance(context.node.config, dict) + assert context.node.config["marker"] == "original" + assert isinstance(context.graph.nodes[1].config, dict) + assert context.graph.nodes[1].config["marker"] == "original" + return context.node.config["marker"] + + result = await start_graph_run( + compiled, + {}, + {"root": mutate_original, "next": observe_fresh}, + run_id="snapshot-isolation", + implementation_id="v1", + event_store=store, + clock=fixed_clock, + ) + + assert result.succeeded + assert dict(result.outputs or {}) == {"result": "original"} + + asyncio.run(scenario()) + + +def test_resume_recomputes_graph_hash_after_nested_graph_mutation() -> None: + async def scenario() -> None: + compiled = graph( + { + "id": "root", + "kind": "agent", + "inputSchema": {}, + "outputSchema": {}, + "config": {"revision": 1}, + "sideEffects": "none", + } + ) + store = MemoryEventStore() + await start_graph_run( + compiled, + {}, + {"root": lambda _: "ok"}, + run_id="mutated-hash", + implementation_id="v1", + event_store=store, + clock=fixed_clock, + ) + config = compiled.spec.nodes[0].config + assert isinstance(config, dict) + config["revision"] = 2 + + with pytest.raises(DurableRunError) as error: + await resume_graph_run( + compiled, + run_id="mutated-hash", + implementation_id="v1", + event_store=store, + ) + + assert error.value.code is DurableRunErrorCode.GRAPH_HASH_MISMATCH + + asyncio.run(scenario()) + + +@pytest.mark.parametrize( + "available_at", + ["2026-07-26T12:00:00", "2026-02-30T12:00:00Z"], +) +def test_retry_available_at_requires_strict_real_rfc3339_before_append( + available_at: str, +) -> None: + retry_graph = graph( + { + "id": "root", + "kind": "agent", + "inputSchema": {}, + "outputSchema": {}, + "config": {}, + "retry": {"maxAttempts": 2}, + "sideEffects": "none", + } + ) + + async def scenario() -> None: + run_id = "bad-time-date" if available_at.endswith("Z") else "bad-time-offset" + source = MemoryEventStore() + calls = 0 + + def flaky(_: NodeContext) -> Any: + nonlocal calls + calls += 1 + if calls == 1: + raise RuntimeError("retry") + return "ok" + + await start_graph_run( + retry_graph, + {}, + {"root": flaky}, + run_id=run_id, + implementation_id="v1", + event_store=source, + clock=fixed_clock, + ) + history = list(await source.read(run_id)) + retry_index = next( + index for index, event in enumerate(history) if event.type == "NodeRetried" + ) + retry_data = dict(history[retry_index].data) + retry_data["availableAt"] = available_at + history[retry_index] = resign(history[retry_index], retry_data) + forged = MemoryEventStore() + await forged.append(run_id, -1, history) + tracked = TrackingStore(forged) + + with pytest.raises(DurableRunError) as error: + await resume_graph_run( + retry_graph, + run_id=run_id, + implementation_id="v1", + event_store=tracked, + ) + + assert error.value.code is DurableRunErrorCode.INVALID_RUN_HISTORY + assert tracked.read_calls == 1 + assert tracked.append_calls == 0 + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("mutation", ["hidden-output", "retryable", "phase", "skipped"]) +def test_terminal_node_outcomes_reject_resigned_semantic_contradictions( + mutation: str, +) -> None: + async def scenario() -> None: + run_id = f"terminal-semantics-{mutation}" + source = MemoryEventStore() + await start_graph_run( + graph(), + {}, + {"root": lambda _: (_ for _ in ()).throw(RuntimeError("failed"))}, + run_id=run_id, + implementation_id="v1", + event_store=source, + clock=fixed_clock, + ) + history = list(await source.read(run_id)) + terminal = history[-1] + result = decode_durable_json(terminal.data["result"]) + assert isinstance(result, dict) + nodes = result["nodes"] + assert isinstance(nodes, list) and isinstance(nodes[0], dict) + failure = nodes[0]["failure"] + assert isinstance(failure, dict) + if mutation == "hidden-output": + nodes[0]["output"] = "forged" + elif mutation == "retryable": + failure["retryable"] = True + elif mutation == "phase": + failure["phase"] = "output" + else: + nodes[0]["status"] = "skipped" + history[-1] = resign(terminal, {"result": encode_durable_json(result)}) + forged = MemoryEventStore() + await forged.append(run_id, -1, history) + + with pytest.raises(DurableRunError) as error: + await resume_graph_run( + graph(), + run_id=run_id, + implementation_id="v1", + event_store=forged, + ) + + assert error.value.code is DurableRunErrorCode.INVALID_RUN_HISTORY + + asyncio.run(scenario()) + + +def test_safe_interrupted_attempt_cannot_be_forged_terminal_while_budget_remains() -> None: + retry_graph = graph( + { + "id": "root", + "kind": "agent", + "inputSchema": {}, + "outputSchema": {}, + "config": {}, + "retry": {"maxAttempts": 2}, + "sideEffects": "none", + } + ) + + async def scenario() -> None: + run_id = "forged-terminal-interruption" + store = MemoryEventStore() + with pytest.raises(ProcessLost): + await start_graph_run( + retry_graph, + {}, + {"root": lambda _: (_ for _ in ()).throw(ProcessLost())}, + run_id=run_id, + implementation_id="v1", + event_store=store, + clock=fixed_clock, + ) + history = await store.read(run_id) + failure = { + "phase": "execute", + "code": "NODE_EXECUTION_INTERRUPTED", + "message": "forged terminal interruption", + "nodeId": "root", + "attempt": 1, + "retryable": False, + "causeName": "ProcessLost", + } + terminal_result = { + "status": "failed", + "graphHash": retry_graph.graph_hash, + "nodes": [ + { + "nodeId": "root", + "sequence": 0, + "status": "failed", + "attempts": 1, + "input": {}, + "failure": failure, + } + ], + "failures": [failure], + "scheduledOrder": ["root"], + "completionOrder": ["root"], + "maxObservedConcurrency": 1, + "totalAttempts": 1, + } + await store.append( + run_id, + history[-1].sequence, + ( + forged_event( + run_id=run_id, + sequence=len(history), + event_type="NodeAttemptFailed", + data={"terminal": True, "failure": failure}, + node_id="root", + attempt=1, + ), + forged_event( + run_id=run_id, + sequence=len(history) + 1, + event_type="RunFailed", + data={"result": encode_durable_json(terminal_result)}, + ), + ), + ) + + with pytest.raises(DurableRunError) as error: + await resume_graph_run( + retry_graph, + run_id=run_id, + implementation_id="v1", + event_store=store, + ) + + assert error.value.code is DurableRunErrorCode.INVALID_RUN_HISTORY + + asyncio.run(scenario()) + + +def test_safe_interrupted_attempt_may_be_terminal_only_after_retry_budget_exhaustion() -> None: + exhausted_graph = graph( + { + "id": "root", + "kind": "agent", + "inputSchema": {}, + "outputSchema": {}, + "config": {}, + "retry": {"maxAttempts": 1}, + "sideEffects": "none", + } + ) + + async def scenario() -> None: + store = MemoryEventStore() + with pytest.raises(ProcessLost): + await start_graph_run( + exhausted_graph, + {}, + {"root": lambda _: (_ for _ in ()).throw(ProcessLost())}, + run_id="exhausted-interruption", + implementation_id="v1", + event_store=store, + clock=fixed_clock, + ) + + result = await resume_graph_run( + exhausted_graph, + run_id="exhausted-interruption", + implementation_id="v1", + event_store=store, + clock=fixed_clock, + ) + assert result.status is RunStatus.FAILED + assert result.nodes["root"].failure is not None + assert result.nodes["root"].failure.code is FailureCode.NODE_EXECUTION_INTERRUPTED + assert not result.nodes["root"].failure.retryable + + assert ( + await resume_graph_run( + exhausted_graph, + run_id="exhausted-interruption", + implementation_id="v1", + event_store=store, + ) + == result + ) + + asyncio.run(scenario()) + + +def test_global_budget_does_not_emit_an_unfunded_single_node_retry() -> None: + limited_graph = graph( + { + "id": "root", + "kind": "agent", + "inputSchema": {}, + "outputSchema": {}, + "config": {}, + "retry": {"maxAttempts": 2}, + "sideEffects": "none", + }, + policies={"maxTotalAttempts": 1}, + ) + + async def scenario() -> None: + store = MemoryEventStore() + result = await start_graph_run( + limited_graph, + {}, + {"root": lambda _: (_ for _ in ()).throw(RuntimeError("failed"))}, + run_id="single-reservation", + implementation_id="v1", + event_store=store, + clock=fixed_clock, + ) + history = await store.read("single-reservation") + + assert result.total_attempts == 1 + assert result.nodes["root"].attempts == 1 + assert not any(event.type == "NodeRetried" for event in history) + attempt_failure = next(event for event in history if event.type == "NodeAttemptFailed") + assert attempt_failure.data["terminal"] is True + assert ( + await resume_graph_run( + limited_graph, + run_id="single-reservation", + implementation_id="v1", + event_store=store, + ) + == result + ) + + asyncio.run(scenario()) + + +def test_concurrent_failures_compete_for_one_global_retry_reservation() -> None: + compiled = compile_graph( + { + "apiVersion": "graphengineering.reacher-z.github.io/v1alpha1", + "kind": "Graph", + "metadata": {"name": "reservation-race", "version": "1"}, + "inputSchema": {}, + "outputSchema": {}, + "entrypoints": ["left", "right"], + "outputs": {"left": {"node": "left"}, "right": {"node": "right"}}, + "nodes": [ + { + "id": node_id, + "kind": "agent", + "inputSchema": {}, + "outputSchema": {}, + "config": {}, + "retry": {"maxAttempts": 2}, + "sideEffects": "none", + } + for node_id in ("left", "right") + ], + "edges": [], + "policies": {"maxConcurrency": 2, "maxTotalAttempts": 3}, + } + ) + + async def scenario() -> None: + store = MemoryEventStore() + both_started = asyncio.Event() + first_attempts = 0 + + async def fail_together(context: NodeContext) -> Any: + nonlocal first_attempts + if context.attempt == 1: + first_attempts += 1 + if first_attempts == 2: + both_started.set() + await both_started.wait() + raise RuntimeError("failed") + + result = await start_graph_run( + compiled, + {}, + {"*": fail_together}, + run_id="reservation-race", + implementation_id="v1", + event_store=store, + clock=fixed_clock, + ) + history = await store.read("reservation-race") + + assert result.status is RunStatus.FAILED + assert result.total_attempts == 3 + assert sorted(node.attempts for node in result.nodes.values()) == [1, 2] + assert sum(event.type == "NodeRetried" for event in history) == 1 + assert ( + sum( + event.type == "NodeAttemptFailed" and event.data["terminal"] is False + for event in history + ) + == 1 + ) + assert ( + await resume_graph_run( + compiled, + run_id="reservation-race", + implementation_id="v1", + event_store=store, + ) + == result + ) + + asyncio.run(scenario()) + + +def test_persisted_retry_reservation_is_counted_before_new_attempt_admission() -> None: + compiled = compile_graph( + { + "apiVersion": "graphengineering.reacher-z.github.io/v1alpha1", + "kind": "Graph", + "metadata": {"name": "persisted-reservation", "version": "1"}, + "inputSchema": {}, + "outputSchema": {}, + "entrypoints": ["a", "b"], + "outputs": {"a": {"node": "a"}, "b": {"node": "b"}}, + "nodes": [ + { + "id": node_id, + "kind": "agent", + "inputSchema": {}, + "outputSchema": {}, + "config": {}, + **( + {"retry": {"maxAttempts": 2}, "sideEffects": "none"} + if node_id == "a" + else {} + ), + } + for node_id in ("a", "b") + ], + "edges": [], + "policies": {"maxConcurrency": 1, "maxTotalAttempts": 2}, + } + ) + + async def scenario() -> None: + run_id = "persisted-reservation" + store = MemoryEventStore() + input_hash = durable_json_hash({}) + activity_key = durable_json_hash(["activity/v1alpha1", run_id, 1, "a", input_hash]) + failure = { + "phase": "execute", + "code": "NODE_EXECUTION_FAILED", + "message": "retry me", + "nodeId": "a", + "attempt": 1, + "retryable": True, + "causeName": "RuntimeError", + } + documents: tuple[tuple[str, dict[str, Any], str | None, int | None], ...] = ( + ( + "RunCreated", + { + "contractVersion": "scheduler-recovery/v1alpha1", + "graphHash": compiled.graph_hash, + "implementationHash": durable_json_hash("v1"), + "input": encode_durable_json({}), + "inputHash": input_hash, + "maxTotalAttempts": 2, + }, + None, + None, + ), + ("RunStarted", {}, None, None), + ( + "NodeScheduled", + { + "input": encode_durable_json({}), + "inputHash": input_hash, + "activityKey": activity_key, + "sideEffects": "none", + }, + "a", + 1, + ), + ( + "NodeStarted", + { + "inputHash": input_hash, + "activityKey": activity_key, + }, + "a", + 1, + ), + ( + "NodeAttemptFailed", + { + "terminal": False, + "failure": failure, + }, + "a", + 1, + ), + ( + "NodeRetried", + { + "availableAt": FIXED_TIME, + "activityKey": activity_key, + }, + "a", + 2, + ), + ( + "NodeSettledWithoutAttempt", + { + "result": encode_durable_json( + { + "nodeId": "b", + "sequence": 1, + "status": "failed", + "attempts": 0, + "input": {}, + "failure": { + "phase": "execute", + "code": "ATTEMPT_BUDGET_EXHAUSTED", + "message": "foreign scheduler budget wording", + "nodeId": "b", + "attempt": 0, + "retryable": False, + }, + } + ) + }, + "b", + None, + ), + ) + await store.append( + run_id, + -1, + tuple( + forged_event( + run_id=run_id, + sequence=index, + event_type=event_type, + data=data, + node_id=node_id, + attempt=attempt, + ) + for index, (event_type, data, node_id, attempt) in enumerate(documents) + ), + ) + b_calls = 0 + + def b_handler(_: NodeContext) -> Any: + nonlocal b_calls + b_calls += 1 + return "must-not-run" + + result = await resume_graph_run( + compiled, + {"a": lambda context: f"a-attempt-{context.attempt}", "b": b_handler}, + run_id=run_id, + implementation_id="v1", + event_store=store, + clock=fixed_clock, + ) + + assert result.total_attempts == 2 + assert result.nodes["a"].attempts == 2 + assert result.nodes["a"].succeeded + assert result.nodes["b"].attempts == 0 + assert result.nodes["b"].failure is not None + assert result.nodes["b"].failure.code is FailureCode.ATTEMPT_BUDGET_EXHAUSTED + assert b_calls == 0 + assert ( + await resume_graph_run( + compiled, + run_id=run_id, + implementation_id="v1", + event_store=store, + ) + == result + ) + + asyncio.run(scenario()) + + +def test_budget_rejection_does_not_inflate_durable_concurrency() -> None: + compiled = compile_graph( + { + "apiVersion": "graphengineering.reacher-z.github.io/v1alpha1", + "kind": "Graph", + "metadata": {"name": "budget-concurrency", "version": "1"}, + "inputSchema": {}, + "outputSchema": {}, + "entrypoints": ["a", "b"], + "outputs": {"a": {"node": "a"}, "b": {"node": "b"}}, + "nodes": [ + { + "id": node_id, + "kind": "agent", + "inputSchema": {}, + "outputSchema": {}, + "config": {}, + "sideEffects": "none", + } + for node_id in ("a", "b") + ], + "edges": [], + "policies": {"maxConcurrency": 2, "maxTotalAttempts": 1}, + } + ) + + async def scenario() -> None: + store = MemoryEventStore() + calls: list[str] = [] + + async def execute(context: NodeContext) -> Any: + calls.append(context.node.id) + await asyncio.sleep(0.01) + return context.node.id + + result = await start_graph_run( + compiled, + {}, + {"*": execute}, + run_id="budget-concurrency", + implementation_id="v1", + event_store=store, + clock=fixed_clock, + ) + history = await store.read("budget-concurrency") + + assert result.max_observed_concurrency == 1 + assert result.total_attempts == 1 + assert len(calls) == 1 + assert sum(event.type == "NodeStarted" for event in history) == 1 + assert ( + await resume_graph_run( + compiled, + run_id="budget-concurrency", + implementation_id="v1", + event_store=store, + ) + == result + ) + + asyncio.run(scenario()) + + +def test_failure_is_committed_before_a_single_slot_admits_the_next_node() -> None: + compiled = compile_graph( + { + "apiVersion": "graphengineering.reacher-z.github.io/v1alpha1", + "kind": "Graph", + "metadata": {"name": "failure-slot-order", "version": "1"}, + "inputSchema": {}, + "outputSchema": {}, + "entrypoints": ["a", "b"], + "outputs": {"a": {"node": "a"}, "b": {"node": "b"}}, + "nodes": [ + { + "id": node_id, + "kind": "agent", + "inputSchema": {}, + "outputSchema": {}, + "config": {}, + "sideEffects": "none", + } + for node_id in ("a", "b") + ], + "edges": [], + "policies": {"maxConcurrency": 1}, + } + ) + + async def scenario() -> None: + store = MemoryEventStore() + result = await start_graph_run( + compiled, + {}, + { + "a": lambda _: (_ for _ in ()).throw(RuntimeError("failed")), + "b": lambda _: "ok", + }, + run_id="failure-slot-order", + implementation_id="v1", + event_store=store, + clock=fixed_clock, + ) + history = await store.read("failure-slot-order") + failed_index = next( + index + for index, event in enumerate(history) + if event.type == "NodeAttemptFailed" and event.node_id == "a" + ) + next_started_index = next( + index + for index, event in enumerate(history) + if event.type == "NodeStarted" and event.node_id == "b" + ) + + assert failed_index < next_started_index + assert result.max_observed_concurrency == 1 + assert ( + await resume_graph_run( + compiled, + run_id="failure-slot-order", + implementation_id="v1", + event_store=store, + ) + == result + ) + + asyncio.run(scenario()) + + +def test_durable_journal_latches_the_first_concurrent_store_failure() -> None: + class FailingStore: + def __init__(self) -> None: + self.append_calls = 0 + + async def read(self, run_id: str, from_sequence: int = 0) -> tuple[GraphEvent, ...]: + return () + + async def append( + self, + run_id: str, + expected_version: int, + values: Sequence[GraphEvent], + ) -> int: + self.append_calls += 1 + await asyncio.sleep(0) + raise RuntimeError("store unavailable") + + async def scenario() -> None: + store = FailingStore() + journal = _DurableJournal( + graph=graph(), + run_id="fatal-latch", + store=store, + version=-1, + clock=fixed_clock, + event_id_factory=lambda run_id, sequence: f"{run_id}:{sequence}", + ) + results = await asyncio.gather( + journal.append([_EventDraft("RunStarted", {})]), + journal.append([_EventDraft("RunStarted", {})]), + return_exceptions=True, + ) + + assert store.append_calls == 1 + assert all(isinstance(item, DurableRunError) for item in results) + first, second = results + assert first is second + assert isinstance(first, DurableRunError) + assert first.code is DurableRunErrorCode.DURABILITY_STORE_FAILED + + asyncio.run(scenario()) + + +def test_durable_journal_failure_does_not_wait_for_handler_ignoring_cancellation() -> None: + compiled = compile_graph( + { + "apiVersion": "graphengineering.reacher-z.github.io/v1alpha1", + "kind": "Graph", + "metadata": {"name": "journal-failure-cancellation", "version": "1"}, + "inputSchema": {}, + "outputSchema": {}, + "entrypoints": ["a", "b"], + "outputs": {"a": {"node": "a"}, "b": {"node": "b"}}, + "nodes": [ + { + "id": node_id, + "kind": "agent", + "inputSchema": {}, + "outputSchema": {}, + "config": {}, + "sideEffects": "none", + } + for node_id in ("a", "b") + ], + "edges": [], + "policies": {"maxConcurrency": 2}, + } + ) + + class FailFirstSuccessStore: + def __init__(self) -> None: + self.delegate = MemoryEventStore() + + async def read( + self, + run_id: str, + from_sequence: int = 0, + ) -> tuple[GraphEvent, ...]: + return await self.delegate.read(run_id, from_sequence) + + async def append( + self, + run_id: str, + expected_version: int, + values: Sequence[GraphEvent], + ) -> int: + if any( + event.type == "NodeSucceeded" and event.node_id == "a" + for event in values + ): + raise RuntimeError("store unavailable") + return await self.delegate.append(run_id, expected_version, values) + + async def scenario() -> None: + loop = asyncio.get_running_loop() + previous_handler = loop.get_exception_handler() + unhandled: list[dict[str, Any]] = [] + loop.set_exception_handler(lambda _loop, context: unhandled.append(context)) + b_started = asyncio.Event() + b_cancelled = asyncio.Event() + release_b = asyncio.Event() + b_finished = asyncio.Event() + + async def a_handler(_: NodeContext) -> Any: + await b_started.wait() + return "a" + + async def b_handler(_: NodeContext) -> Any: + b_started.set() + try: + await asyncio.Future() + except asyncio.CancelledError: + b_cancelled.set() + await release_b.wait() + raise RuntimeError("late handler failure") from None + finally: + b_finished.set() + + try: + run = asyncio.create_task( + start_graph_run( + compiled, + {}, + {"a": a_handler, "b": b_handler}, + run_id="journal-failure-cancellation", + implementation_id="v1", + event_store=FailFirstSuccessStore(), + clock=fixed_clock, + ) + ) + with pytest.raises(DurableRunError) as error: + await asyncio.wait_for(asyncio.shield(run), timeout=0.25) + assert error.value.code is DurableRunErrorCode.DURABILITY_STORE_FAILED + await asyncio.wait_for(b_cancelled.wait(), timeout=0.25) + + release_b.set() + await asyncio.wait_for(b_finished.wait(), timeout=0.25) + for _ in range(3): + await asyncio.sleep(0) + assert unhandled == [] + finally: + release_b.set() + loop.set_exception_handler(previous_handler) + + asyncio.run(scenario()) + + +def test_durable_journal_rejects_and_latches_an_inexact_returned_version() -> None: + class WrongVersionStore: + def __init__(self) -> None: + self.append_calls = 0 + + async def read(self, run_id: str, from_sequence: int = 0) -> tuple[GraphEvent, ...]: + return () + + async def append( + self, + run_id: str, + expected_version: int, + values: Sequence[GraphEvent], + ) -> int: + self.append_calls += 1 + return expected_version + + async def scenario() -> None: + store = WrongVersionStore() + journal = _DurableJournal( + graph=graph(), + run_id="wrong-version", + store=store, + version=-1, + clock=fixed_clock, + event_id_factory=lambda run_id, sequence: f"{run_id}:{sequence}", + ) + with pytest.raises(DurableRunError) as first: + await journal.append([_EventDraft("RunStarted", {})]) + with pytest.raises(DurableRunError) as second: + await journal.append([_EventDraft("RunStarted", {})]) + + assert first.value.code is DurableRunErrorCode.DURABILITY_STORE_FAILED + assert first.value.details == {"expectedVersion": 0, "actualVersion": -1} + assert second.value is first.value + assert store.append_calls == 1 + + asyncio.run(scenario()) + + +def test_missing_executor_commits_explicit_outcome_and_terminal_resume_is_read_only() -> None: + async def scenario() -> None: + store = MemoryEventStore() + result = await start_graph_run( + graph(), + {"realNull": None}, + run_id="missing-executor-outcome", + implementation_id="v1", + event_store=store, + clock=fixed_clock, + ) + history = await store.read("missing-executor-outcome") + + assert [event.type for event in history] == [ + "RunCreated", + "RunStarted", + "NodeSettledWithoutAttempt", + "RunFailed", + ] + settled = decode_durable_json(history[2].data["result"]) + assert isinstance(settled, dict) + assert settled["attempts"] == 0 + assert settled["input"] == {"realNull": None} + assert settled["failure"]["code"] == "EXECUTOR_NOT_FOUND" + + tracked = TrackingStore(store) + calls = 0 + + def must_not_run(_: NodeContext) -> Any: + nonlocal calls + calls += 1 + return "forged" + + resumed = await resume_graph_run( + graph(), + {"root": must_not_run}, + run_id="missing-executor-outcome", + implementation_id="v1", + event_store=tracked, + ) + assert resumed == result + assert calls == 0 + assert tracked.read_calls == 1 + assert tracked.append_calls == 0 + + asyncio.run(scenario()) + + +def test_node_settlement_commits_before_a_failed_descendant_is_released() -> None: + compiled = compile_graph( + { + "apiVersion": "graphengineering.reacher-z.github.io/v1alpha1", + "kind": "Graph", + "metadata": {"name": "settle-before-release", "version": "1"}, + "inputSchema": {}, + "outputSchema": {}, + "entrypoints": ["root"], + "outputs": {"result": {"node": "child"}}, + "nodes": [ + { + "id": node_id, + "kind": "agent", + "inputSchema": {}, + "outputSchema": {}, + "config": {}, + "sideEffects": "none", + } + for node_id in ("root", "child") + ], + "edges": [{"id": "root-child", "from": {"node": "root"}, "to": {"node": "child"}}], + } + ) + + class GateStore: + def __init__(self) -> None: + self.delegate = MemoryEventStore() + self.root_settle_entered = asyncio.Event() + self.release_root_settle = asyncio.Event() + self.appended_node_ids: list[str | None] = [] + + async def read(self, run_id: str, from_sequence: int = 0) -> tuple[GraphEvent, ...]: + return await self.delegate.read(run_id, from_sequence) + + async def append( + self, + run_id: str, + expected_version: int, + values: Sequence[GraphEvent], + ) -> int: + first = values[0] + if first.type == "NodeSettledWithoutAttempt": + self.appended_node_ids.append(first.node_id) + if first.type == "NodeSettledWithoutAttempt" and first.node_id == "root": + self.root_settle_entered.set() + await self.release_root_settle.wait() + return await self.delegate.append(run_id, expected_version, values) + + async def scenario() -> None: + store = GateStore() + task = asyncio.create_task( + start_graph_run( + compiled, + {}, + run_id="settle-before-release", + implementation_id="v1", + event_store=store, + clock=fixed_clock, + ) + ) + await store.root_settle_entered.wait() + assert store.appended_node_ids == ["root"] + assert [event.type for event in await store.delegate.read("settle-before-release")] == [ + "RunCreated", + "RunStarted", + ] + store.release_root_settle.set() + result = await task + history = await store.delegate.read("settle-before-release") + outcomes = [event.node_id for event in history if event.type == "NodeSettledWithoutAttempt"] + assert outcomes == ["root", "child"] + child_event = next( + event + for event in history + if event.type == "NodeSettledWithoutAttempt" and event.node_id == "child" + ) + child_document = decode_durable_json(child_event.data["result"]) + assert isinstance(child_document, dict) and "input" not in child_document + assert result.nodes["child"].attempts == 0 + + asyncio.run(scenario()) + + +def test_retry_delay_cancellation_settles_with_the_preserved_attempt_offset() -> None: + retry_graph = graph( + { + "id": "root", + "kind": "agent", + "inputSchema": {}, + "outputSchema": {}, + "config": {}, + "retry": {"maxAttempts": 2, "initialDelayMs": 1000}, + "sideEffects": "none", + } + ) + + async def scenario() -> None: + store = MemoryEventStore() + cancel = asyncio.Event() + + def fail_then_cancel(_: NodeContext) -> Any: + asyncio.get_running_loop().call_soon(cancel.set) + raise RuntimeError("retryable") + + result = await start_graph_run( + retry_graph, + None, + {"root": fail_then_cancel}, + run_id="retry-delay-cancel", + implementation_id="v1", + event_store=store, + cancel_event=cancel, + clock=fixed_clock, + ) + history = await store.read("retry-delay-cancel") + settled_event = next( + event for event in history if event.type == "NodeSettledWithoutAttempt" + ) + settled = decode_durable_json(settled_event.data["result"]) + assert isinstance(settled, dict) + assert settled["attempts"] == 1 + assert "input" in settled and settled["input"] is None + assert settled["failure"]["code"] == "NODE_CANCELLED" + assert result.total_attempts == 1 + assert result.status is RunStatus.CANCELLED + assert ( + await resume_graph_run( + retry_graph, + run_id="retry-delay-cancel", + implementation_id="v1", + event_store=store, + ) + == result + ) + + asyncio.run(scenario()) + + +def test_start_and_resume_snapshot_handler_registries_before_store_io() -> None: + class BlockingReadStore: + def __init__(self, delegate: MemoryEventStore) -> None: + self.delegate = delegate + self.read_entered = asyncio.Event() + self.release_read = asyncio.Event() + + async def read(self, run_id: str, from_sequence: int = 0) -> tuple[GraphEvent, ...]: + self.read_entered.set() + await self.release_read.wait() + return await self.delegate.read(run_id, from_sequence) + + async def append( + self, + run_id: str, + expected_version: int, + values: Sequence[GraphEvent], + ) -> int: + return await self.delegate.append(run_id, expected_version, values) + + async def scenario() -> None: + start_delegate = MemoryEventStore() + start_store = BlockingReadStore(start_delegate) + start_calls: list[str] = [] + registry = {"root": lambda _: start_calls.append("original") or "original"} + start_task = asyncio.create_task( + start_graph_run( + graph(), + {}, + registry, + run_id="handler-snapshot-start", + implementation_id="v1", + event_store=start_store, + clock=fixed_clock, + ) + ) + await start_store.read_entered.wait() + registry["root"] = lambda _: start_calls.append("replacement") or "replacement" + start_store.release_read.set() + started = await start_task + assert start_calls == ["original"] + assert dict(started.outputs or {}) == {"result": "original"} + + retry_graph = graph( + { + "id": "root", + "kind": "agent", + "inputSchema": {}, + "outputSchema": {}, + "config": {}, + "retry": {"maxAttempts": 2}, + "sideEffects": "none", + } + ) + resume_delegate = MemoryEventStore() + with pytest.raises(ProcessLost): + await start_graph_run( + retry_graph, + {}, + {"root": lambda _: (_ for _ in ()).throw(ProcessLost())}, + run_id="handler-snapshot-resume", + implementation_id="v1", + event_store=resume_delegate, + clock=fixed_clock, + ) + resume_store = BlockingReadStore(resume_delegate) + resume_calls: list[str] = [] + resume_registry = {"root": lambda _: resume_calls.append("original") or "original"} + resume_task = asyncio.create_task( + resume_graph_run( + retry_graph, + resume_registry, + run_id="handler-snapshot-resume", + implementation_id="v1", + event_store=resume_store, + clock=fixed_clock, + ) + ) + await resume_store.read_entered.wait() + resume_registry["root"] = lambda _: resume_calls.append("replacement") or "replacement" + resume_store.release_read.set() + resumed = await resume_task + assert resume_calls == ["original"] + assert dict(resumed.outputs or {}) == {"result": "original"} + + asyncio.run(scenario()) + + +def test_each_attempt_gets_a_fresh_graph_context_and_cannot_corrupt_history() -> None: + compiled = compile_graph( + { + "apiVersion": "graphengineering.reacher-z.github.io/v1alpha1", + "kind": "Graph", + "metadata": {"name": "attempt-context-isolation", "version": "1"}, + "inputSchema": {}, + "outputSchema": {}, + "entrypoints": ["root"], + "outputs": {"result": {"node": "child"}}, + "nodes": [ + { + "id": "root", + "kind": "agent", + "inputSchema": {}, + "outputSchema": {}, + "config": {"marker": "root-original"}, + "retry": {"maxAttempts": 2}, + "sideEffects": "none", + }, + { + "id": "child", + "kind": "agent", + "inputSchema": {}, + "outputSchema": {}, + "config": {"marker": "child-original"}, + "sideEffects": "none", + }, + ], + "edges": [{"id": "root-child", "from": {"node": "root"}, "to": {"node": "child"}}], + } + ) + + async def scenario() -> None: + store = MemoryEventStore() + + def root(context: NodeContext) -> Any: + assert isinstance(context.node.config, dict) + assert context.node.config["marker"] == "root-original" + child_config = context.graph.nodes[1].config + assert isinstance(child_config, dict) + child_config["marker"] = "poisoned" + context.graph.outputs.clear() + context.node.config["marker"] = "attempt-local" + if context.attempt == 1: + raise RuntimeError("retry with a fresh graph") + return "root-ok" + + def child(context: NodeContext) -> Any: + assert isinstance(context.node.config, dict) + assert context.node.config["marker"] == "child-original" + assert "result" in context.graph.outputs + return "child-ok" + + result = await start_graph_run( + compiled, + {}, + {"root": root, "child": child}, + run_id="attempt-context-isolation", + implementation_id="v1", + event_store=store, + clock=fixed_clock, + ) + assert result.succeeded + assert result.nodes["root"].attempts == 2 + assert dict(result.outputs or {}) == {"result": "child-ok"} + assert ( + await resume_graph_run( + compiled, + run_id="attempt-context-isolation", + implementation_id="v1", + event_store=store, + ) + == result + ) + + asyncio.run(scenario()) + + +def test_durable_completion_order_uses_outcome_commit_ordinals() -> None: + compiled = compile_graph( + { + "apiVersion": "graphengineering.reacher-z.github.io/v1alpha1", + "kind": "Graph", + "metadata": {"name": "commit-order", "version": "1"}, + "inputSchema": {}, + "outputSchema": {}, + "entrypoints": ["a", "b"], + "outputs": {"a": {"node": "a"}, "b": {"node": "b"}}, + "nodes": [ + { + "id": node_id, + "kind": "agent", + "inputSchema": {}, + "outputSchema": {}, + "config": {}, + "sideEffects": "none", + } + for node_id in ("a", "b") + ], + "edges": [], + } + ) + + class CoordinatedJournal: + def __init__(self) -> None: + self.committed: set[str] = set() + self.both_committed = asyncio.Event() + self.terminal_completion: tuple[str, ...] | None = None + + async def before_attempt(self, node: Any, node_input: Any, attempt: int) -> Any: + return _AttemptIdentity("commit-order", f"{node.id}/{attempt}", node.id) + + async def node_succeeded( + self, node: Any, node_input: Any, attempt: int, output: Any + ) -> int: + self.committed.add(node.id) + if len(self.committed) == 2: + self.both_committed.set() + await self.both_committed.wait() + return 1 if node.id == "b" else 2 + + async def attempt_failed(self, *args: Any, **kwargs: Any) -> int: + raise AssertionError("unexpected attempt failure") + + async def node_settled_without_attempt(self, *args: Any, **kwargs: Any) -> int: + raise AssertionError("unexpected settlement") + + async def run_terminal(self, result: Any) -> None: + self.terminal_completion = result.completion_order + + async def scenario() -> None: + journal = CoordinatedJournal() + result = await AsyncScheduler({"*": lambda context: context.node.id})._run( + compiled, + {}, + journal=journal, + ) + assert result.completion_order == ("b", "a") + assert journal.terminal_completion == ("b", "a") + + asyncio.run(scenario()) + + +def test_output_binding_failure_uses_canonical_shape_and_semantic_message_matching() -> None: + output_graph = compile_graph( + { + "apiVersion": "graphengineering.reacher-z.github.io/v1alpha1", + "kind": "Graph", + "metadata": {"name": "output-failure-shape", "version": "1"}, + "inputSchema": {}, + "outputSchema": {}, + "entrypoints": ["root"], + "outputs": {"answer": {"node": "root", "port": "missing"}}, + "nodes": [ + { + "id": "root", + "kind": "agent", + "inputSchema": {}, + "outputSchema": {}, + "config": {}, + "sideEffects": "none", + } + ], + "edges": [], + } + ) + + async def scenario() -> None: + source = MemoryEventStore() + result = await start_graph_run( + output_graph, + {}, + {"root": lambda _: {}}, + run_id="output-failure-shape", + implementation_id="v1", + event_store=source, + clock=fixed_clock, + ) + history = list(await source.read("output-failure-shape")) + terminal = history[-1] + document = decode_durable_json(terminal.data["result"]) + assert isinstance(document, dict) + failures = document["failures"] + assert isinstance(failures, list) and isinstance(failures[0], dict) + failure = failures[0] + assert set(failure) == { + "phase", + "code", + "message", + "outputName", + "nodeId", + "port", + } + assert failure["port"] == "missing" + assert result.failures[-1].output_port == "missing" + + failure["message"] = "Foreign runtime wording is allowed" + history[-1] = resign(terminal, {"result": encode_durable_json(document)}) + foreign = MemoryEventStore() + await foreign.append("output-failure-shape", -1, history) + resumed = await resume_graph_run( + output_graph, + run_id="output-failure-shape", + implementation_id="v1", + event_store=foreign, + ) + assert resumed.failures[-1].message == "Foreign runtime wording is allowed" + + del failure["port"] + history[-1] = resign(terminal, {"result": encode_durable_json(document)}) + missing_port = MemoryEventStore() + await missing_port.append("output-failure-shape", -1, history) + with pytest.raises(DurableRunError) as error: + await resume_graph_run( + output_graph, + run_id="output-failure-shape", + implementation_id="v1", + event_store=missing_port, + ) + assert error.value.code is DurableRunErrorCode.INVALID_RUN_HISTORY + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("mutation", ["duplicate-event-id", "terminal-node-identity"]) +def test_fold_rejects_duplicate_event_ids_and_extraneous_terminal_identity( + mutation: str, +) -> None: + async def scenario() -> None: + source = MemoryEventStore() + await start_graph_run( + graph(), + {}, + {"root": lambda _: "ok"}, + run_id=f"event-identity-{mutation}", + implementation_id="v1", + event_store=source, + clock=fixed_clock, + ) + history = list(await source.read(f"event-identity-{mutation}")) + if mutation == "duplicate-event-id": + history[-1] = history[-1].model_copy(update={"event_id": history[0].event_id}) + else: + history[-1] = history[-1].model_copy(update={"node_id": "root"}) + forged = MemoryEventStore() + await forged.append(f"event-identity-{mutation}", -1, history) + + with pytest.raises(DurableRunError) as error: + await resume_graph_run( + graph(), + run_id=f"event-identity-{mutation}", + implementation_id="v1", + event_store=forged, + ) + assert error.value.code is DurableRunErrorCode.INVALID_RUN_HISTORY + + asyncio.run(scenario()) + + +def test_writer_rejects_duplicate_event_ids_before_any_store_write() -> None: + async def scenario() -> None: + store = TrackingStore() + with pytest.raises(DurableRunError) as error: + await start_graph_run( + graph(), + {}, + {"root": lambda _: "ok"}, + run_id="duplicate-writer-id", + implementation_id="v1", + event_store=store, + event_id_factory=lambda _run_id, _sequence: "constant", + clock=fixed_clock, + ) + assert error.value.code is DurableRunErrorCode.DURABILITY_STORE_FAILED + assert store.append_calls == 0 + assert await store.read("duplicate-writer-id") == () + + asyncio.run(scenario()) + + +def test_resume_writer_rejects_an_event_id_that_collides_with_history() -> None: + retry_graph = graph( + { + "id": "root", + "kind": "agent", + "inputSchema": {}, + "outputSchema": {}, + "config": {}, + "retry": {"maxAttempts": 2}, + "sideEffects": "none", + } + ) + + async def scenario() -> None: + delegate = MemoryEventStore() + with pytest.raises(ProcessLost): + await start_graph_run( + retry_graph, + {}, + {"root": lambda _: (_ for _ in ()).throw(ProcessLost())}, + run_id="resume-id-collision", + implementation_id="v1", + event_store=delegate, + clock=fixed_clock, + ) + store = TrackingStore(delegate) + calls = 0 + + def handler(_: NodeContext) -> Any: + nonlocal calls + calls += 1 + return "must-not-run" + + with pytest.raises(DurableRunError) as error: + await resume_graph_run( + retry_graph, + {"root": handler}, + run_id="resume-id-collision", + implementation_id="v1", + event_store=store, + event_id_factory=lambda _run_id, _sequence: "resume-id-collision:0", + clock=fixed_clock, + ) + assert error.value.code is DurableRunErrorCode.DURABILITY_STORE_FAILED + assert store.append_calls == 0 + assert calls == 0 + + asyncio.run(scenario()) + + +def test_retry_timestamp_overflow_is_mapped_and_fatal_latched() -> None: + retry_graph = graph( + { + "id": "root", + "kind": "agent", + "inputSchema": {}, + "outputSchema": {}, + "config": {}, + "retry": {"maxAttempts": 2, "initialDelayMs": 1}, + "sideEffects": "none", + } + ) + + async def scenario() -> None: + store = TrackingStore() + journal = _DurableJournal( + graph=retry_graph, + run_id="clock-overflow", + store=store, + version=-1, + clock=lambda: "9999-12-31T23:59:59.999Z", + event_id_factory=lambda run_id, sequence: f"{run_id}:{sequence}", + ) + await journal.before_attempt(retry_graph.nodes["root"], {}, 1) + failure = NodeFailure( + FailureCode.NODE_EXECUTION_FAILED, + "retry", + "root", + 1, + retryable=True, + ) + with pytest.raises(DurableRunError) as first: + await journal.attempt_failed( + failure, + will_retry=True, + retry_delay_ms=1, + ) + with pytest.raises(DurableRunError) as second: + await journal.append([_EventDraft("RunStarted", {})]) + assert first.value.code is DurableRunErrorCode.DURABILITY_STORE_FAILED + assert second.value is first.value + assert store.append_calls == 1 + + asyncio.run(scenario()) + + +def test_terminal_event_cannot_supply_an_uncommitted_node_outcome() -> None: + async def scenario() -> None: + source = MemoryEventStore() + await start_graph_run( + graph(), + {}, + run_id="terminal-without-outcome", + implementation_id="v1", + event_store=source, + clock=fixed_clock, + ) + history = list(await source.read("terminal-without-outcome")) + terminal = history[-1].model_copy( + update={"sequence": 2, "event_id": "terminal-without-outcome:terminal"} + ) + forged = MemoryEventStore() + await forged.append("terminal-without-outcome", -1, (*history[:2], terminal)) + + with pytest.raises(DurableRunError) as error: + await resume_graph_run( + graph(), + run_id="terminal-without-outcome", + implementation_id="v1", + event_store=forged, + ) + assert error.value.code is DurableRunErrorCode.INVALID_RUN_HISTORY + + asyncio.run(scenario()) + + +def test_attempt_event_rejects_a_non_attempt_failure_code() -> None: + async def scenario() -> None: + source = MemoryEventStore() + await start_graph_run( + graph(), + {}, + {"root": lambda _: (_ for _ in ()).throw(RuntimeError("failed"))}, + run_id="forged-attempt-code", + implementation_id="v1", + event_store=source, + clock=fixed_clock, + ) + history = list(await source.read("forged-attempt-code")) + index = next(i for i, event in enumerate(history) if event.type == "NodeAttemptFailed") + data = dict(history[index].data) + failure = dict(data["failure"]) + failure["code"] = "EXECUTOR_NOT_FOUND" + data["failure"] = failure + history[index] = resign(history[index], data) + forged = MemoryEventStore() + await forged.append("forged-attempt-code", -1, history) + + with pytest.raises(DurableRunError) as error: + await resume_graph_run( + graph(), + run_id="forged-attempt-code", + implementation_id="v1", + event_store=forged, + ) + assert error.value.code is DurableRunErrorCode.INVALID_RUN_HISTORY + + asyncio.run(scenario()) + + +def test_settled_node_cannot_be_started_again_after_success() -> None: + async def scenario() -> None: + source = MemoryEventStore() + await start_graph_run( + graph(), + {}, + {"root": lambda _: "ok"}, + run_id="second-start", + implementation_id="v1", + event_store=source, + clock=fixed_clock, + ) + history = list(await source.read("second-start")) + started = next(event for event in history if event.type == "NodeStarted") + terminal = history[-1] + second_start = forged_event( + run_id="second-start", + sequence=terminal.sequence, + event_type="NodeStarted", + data=dict(started.data), + node_id="root", + attempt=1, + ) + shifted_terminal = terminal.model_copy( + update={ + "sequence": terminal.sequence + 1, + "event_id": "second-start:shifted-terminal", + } + ) + forged = MemoryEventStore() + await forged.append( + "second-start", + -1, + (*history[:-1], second_start, shifted_terminal), + ) + + with pytest.raises(DurableRunError) as error: + await resume_graph_run( + graph(), + run_id="second-start", + implementation_id="v1", + event_store=forged, + ) + assert error.value.code is DurableRunErrorCode.INVALID_RUN_HISTORY + + asyncio.run(scenario()) + + +def test_retry_reservation_cannot_be_consumed_by_restarting_the_old_attempt() -> None: + retry_graph = graph( + { + "id": "root", + "kind": "agent", + "inputSchema": {}, + "outputSchema": {}, + "config": {}, + "retry": {"maxAttempts": 2}, + "sideEffects": "none", + } + ) + + async def scenario() -> None: + source = MemoryEventStore() + calls = 0 + + def flaky(_: NodeContext) -> Any: + nonlocal calls + calls += 1 + if calls == 1: + raise RuntimeError("retry") + return "ok" + + await start_graph_run( + retry_graph, + {}, + {"root": flaky}, + run_id="old-attempt-restart", + implementation_id="v1", + event_store=source, + clock=fixed_clock, + ) + history = list(await source.read("old-attempt-restart")) + retry_index = next(i for i, event in enumerate(history) if event.type == "NodeRetried") + first_started = next(event for event in history if event.type == "NodeStarted") + forged_old_start = forged_event( + run_id="old-attempt-restart", + sequence=retry_index + 1, + event_type="NodeStarted", + data=dict(first_started.data), + node_id="root", + attempt=1, + ) + forged = MemoryEventStore() + await forged.append( + "old-attempt-restart", + -1, + (*history[: retry_index + 1], forged_old_start), + ) + + with pytest.raises(DurableRunError) as error: + await resume_graph_run( + retry_graph, + run_id="old-attempt-restart", + implementation_id="v1", + event_store=forged, + ) + assert error.value.code is DurableRunErrorCode.INVALID_RUN_HISTORY + + asyncio.run(scenario()) + + +def test_downstream_cannot_settle_before_its_upstream_has_an_outcome() -> None: + compiled = compile_graph( + { + "apiVersion": "graphengineering.reacher-z.github.io/v1alpha1", + "kind": "Graph", + "metadata": {"name": "premature-settlement", "version": "1"}, + "inputSchema": {}, + "outputSchema": {}, + "entrypoints": ["root"], + "outputs": {"result": {"node": "child"}}, + "nodes": [ + { + "id": node_id, + "kind": "agent", + "inputSchema": {}, + "outputSchema": {}, + "config": {}, + "sideEffects": "none", + } + for node_id in ("root", "child") + ], + "edges": [{"id": "root-child", "from": {"node": "root"}, "to": {"node": "child"}}], + } + ) + + async def scenario() -> None: + source = MemoryEventStore() + await start_graph_run( + compiled, + {}, + run_id="premature-settlement", + implementation_id="v1", + event_store=source, + clock=fixed_clock, + ) + history = list(await source.read("premature-settlement")) + root = next( + event + for event in history + if event.type == "NodeSettledWithoutAttempt" and event.node_id == "root" + ) + child = next( + event + for event in history + if event.type == "NodeSettledWithoutAttempt" and event.node_id == "child" + ) + premature_child = child.model_copy( + update={"sequence": 2, "event_id": "premature-settlement:child-first"} + ) + late_root = root.model_copy( + update={"sequence": 3, "event_id": "premature-settlement:root-second"} + ) + forged = MemoryEventStore() + await forged.append( + "premature-settlement", + -1, + (*history[:2], premature_child, late_root), + ) + + with pytest.raises(DurableRunError) as error: + await resume_graph_run( + compiled, + run_id="premature-settlement", + implementation_id="v1", + event_store=forged, + ) + assert error.value.code is DurableRunErrorCode.INVALID_RUN_HISTORY + + asyncio.run(scenario()) + + +def test_terminal_event_rejects_a_pending_retry_reservation() -> None: + retry_graph = graph( + { + "id": "root", + "kind": "agent", + "inputSchema": {}, + "outputSchema": {}, + "config": {}, + "retry": {"maxAttempts": 2}, + "sideEffects": "none", + } + ) + + async def scenario() -> None: + source = MemoryEventStore() + calls = 0 + + def flaky(_: NodeContext) -> Any: + nonlocal calls + calls += 1 + if calls == 1: + raise RuntimeError("retry") + return "ok" + + await start_graph_run( + retry_graph, + {}, + {"root": flaky}, + run_id="terminal-pending-reservation", + implementation_id="v1", + event_store=source, + clock=fixed_clock, + ) + history = list(await source.read("terminal-pending-reservation")) + retry_index = next(i for i, event in enumerate(history) if event.type == "NodeRetried") + terminal = history[-1].model_copy( + update={ + "sequence": retry_index + 1, + "event_id": "terminal-pending-reservation:early-terminal", + } + ) + forged = MemoryEventStore() + await forged.append( + "terminal-pending-reservation", + -1, + (*history[: retry_index + 1], terminal), + ) + + with pytest.raises(DurableRunError) as error: + await resume_graph_run( + retry_graph, + run_id="terminal-pending-reservation", + implementation_id="v1", + event_store=forged, + ) + assert error.value.code is DurableRunErrorCode.INVALID_RUN_HISTORY + + asyncio.run(scenario()) diff --git a/python/tests/test_events.py b/python/tests/test_events.py index 8e589bf..5a62845 100644 --- a/python/tests/test_events.py +++ b/python/tests/test_events.py @@ -11,6 +11,17 @@ ROOT = Path(__file__).resolve().parents[2] +def test_graph_event_consumes_shared_strict_rfc3339_corpus() -> None: + corpus = json.loads((ROOT / "spec/conformance/strict-rfc3339.case.json").read_text()) + base = json.loads((ROOT / "spec/conformance/run-created.event.json").read_text()) + + for timestamp in corpus["valid"]: + GraphEvent.model_validate({**base, "timestamp": timestamp}) + for timestamp in corpus["invalid"]: + with pytest.raises(ValidationError): + GraphEvent.model_validate({**base, "timestamp": timestamp}) + + def test_graph_event_matches_shared_conformance_envelope() -> None: document = json.loads((ROOT / "spec/conformance/run-created.event.json").read_text()) diff --git a/python/tests/test_scheduler.py b/python/tests/test_scheduler.py index 764fa66..8baf191 100644 --- a/python/tests/test_scheduler.py +++ b/python/tests/test_scheduler.py @@ -194,6 +194,77 @@ def always_fails(_: NodeContext) -> Any: assert not result.nodes["root"].failure.retryable +def test_restored_attempt_offsets_are_preserved_without_another_attempt() -> None: + resumable = compile_graph( + make_graph(nodes=[make_node("root", kind="agent", retry={"maxAttempts": 3})]) + ) + + missing = asyncio.run( + AsyncScheduler()._run( + resumable, + {}, + attempt_offsets={"root": 2}, + initial_total_attempts=2, + ) + ) + assert missing.nodes["root"].attempts == 2 + assert missing.nodes["root"].failure is not None + assert missing.nodes["root"].failure.attempt == 2 + assert missing.nodes["root"].failure.code is FailureCode.EXECUTOR_NOT_FOUND + + cancelled_event = asyncio.Event() + cancelled_event.set() + cancelled = asyncio.run( + AsyncScheduler({"root": lambda _: "must-not-run"})._run( + resumable, + {}, + cancel_event=cancelled_event, + attempt_offsets={"root": 2}, + initial_total_attempts=2, + ) + ) + assert cancelled.nodes["root"].attempts == 2 + assert cancelled.nodes["root"].failure is not None + assert cancelled.nodes["root"].failure.attempt == 2 + assert cancelled.nodes["root"].failure.code is FailureCode.NODE_CANCELLED + + exhausted = compile_graph( + make_graph(nodes=[make_node("root", kind="agent", retry={"maxAttempts": 2})]) + ) + budget = asyncio.run( + AsyncScheduler({"root": lambda _: "must-not-run"})._run( + exhausted, + {}, + attempt_offsets={"root": 2}, + initial_total_attempts=2, + ) + ) + assert budget.nodes["root"].attempts == 2 + assert budget.nodes["root"].failure is not None + assert budget.nodes["root"].failure.attempt == 2 + assert budget.nodes["root"].failure.code is FailureCode.ATTEMPT_BUDGET_EXHAUSTED + + +def test_retry_delay_is_capped_without_overflow_and_never_below_initial_delay() -> None: + compiled = compile_graph( + make_graph( + nodes=[ + make_node( + "root", + retry={ + "maxAttempts": 100, + "initialDelayMs": 10, + "maxDelayMs": 1, + "backoffMultiplier": float(2**53 - 1), + }, + ) + ] + ) + ) + + assert AsyncScheduler._retry_delay_ms(compiled.nodes["root"], 100) == 10 + + def test_timeout_is_structured() -> None: graph = compile_graph(make_graph(nodes=[make_node("root", timeoutMs=2)])) diff --git a/scripts/validate-fixtures.mjs b/scripts/validate-fixtures.mjs index 5f99c82..4d2aa3a 100644 --- a/scripts/validate-fixtures.mjs +++ b/scripts/validate-fixtures.mjs @@ -34,6 +34,40 @@ function hash(value) { return createHash("sha256").update(serialized, "utf8").digest("hex"); } +function floatFromBits(bits) { + const bytes = Buffer.from(bits, "hex"); + assert.equal(bytes.length, 8, `expected one binary64 value, received ${bits}`); + return bytes.readDoubleBE(0); +} + +function floatBits(value) { + const bytes = Buffer.allocUnsafe(8); + bytes.writeDoubleBE(value, 0); + return bytes.toString("hex"); +} + +function encodeDurableJson(value) { + if (value === null) return ["n"]; + if (typeof value === "boolean") return ["b", value]; + if (typeof value === "string") return ["s", value]; + if (typeof value === "number") { + assert.ok(Number.isFinite(value), "Durable JSON source numbers must be finite"); + if (Number.isInteger(value)) { + assert.ok(Number.isSafeInteger(value), "Durable JSON source integers must be safe"); + return ["i", Object.is(value, -0) ? 0 : value]; + } + return ["f", floatBits(value)]; + } + if (Array.isArray(value)) return ["a", value.map(encodeDurableJson)]; + assert.equal(typeof value, "object"); + return [ + "o", + Object.keys(value) + .sort(compareUnicodeCodePoints) + .map((key) => [key, encodeDurableJson(value[key])]), + ]; +} + async function loadJson(name) { return JSON.parse(await readFile(join(fixtureRoot, name), "utf8")); } @@ -56,6 +90,26 @@ for (const [name, expectation] of Object.entries(expected.checkpoints ?? {})) { assert.equal(hash(body), expectation.contentHash, `${name} checkpoint body hash drifted`); } +const durableJson = await loadJson("durable-json.case.json"); +for (const testCase of durableJson.validCases) { + const source = testCase.source.kind === "float64Bits" + ? floatFromBits(testCase.source.bits) + : testCase.source.value; + const encoding = encodeDurableJson(source); + const canonicalJson = JSON.stringify(encoding); + assert.deepEqual(encoding, testCase.expect.encoding, `${testCase.name} Durable JSON encoding drifted`); + assert.equal( + canonicalJson, + testCase.expect.canonicalJson, + `${testCase.name} Durable JSON canonical text drifted`, + ); + assert.equal( + createHash("sha256").update(canonicalJson, "utf8").digest("hex"), + testCase.expect.sha256, + `${testCase.name} Durable JSON hash drifted`, + ); +} + const diamond = await loadJson("diamond.graph.json"); assert.equal(diamond.apiVersion, "graphengineering.reacher-z.github.io/v1alpha1"); assert.equal(diamond.kind, "Graph"); @@ -63,5 +117,5 @@ assert.equal(new Set(diamond.nodes.map(({ id }) => id)).size, diamond.nodes.leng assert.equal(new Set(diamond.edges.map(({ id }) => id)).size, diamond.edges.length); process.stdout.write( - `Validated ${fixtureNames.length} JSON fixtures, ${Object.keys(expected.canonicalization).length} graph hash, and ${Object.keys(expected.checkpoints ?? {}).length} checkpoint hash.\n`, + `Validated ${fixtureNames.length} JSON fixtures, ${Object.keys(expected.canonicalization).length} graph hash, ${Object.keys(expected.checkpoints ?? {}).length} checkpoint hash, and ${durableJson.validCases.length} Durable JSON vectors.\n`, ); diff --git a/spec/README.md b/spec/README.md index 4d2e8d1..dfeebd0 100644 --- a/spec/README.md +++ b/spec/README.md @@ -3,6 +3,8 @@ `graph.schema.json` is the language-neutral Graph IR contract. `event.schema.json` defines the portable runtime event envelope. `checkpoint.schema.json` defines the portable stored-checkpoint envelope. +`durable-json.schema.json` defines the tagged, checkpoint-safe encoding used by +scheduler recovery for portable finite JSON, including exact binary64 values. `conformance/` contains inputs and expected results used by every native runtime, including settled-barrier and deterministic route-selection decision corpora. @@ -10,6 +12,8 @@ Runtime scheduling is fixed by [runtime-semantics.md](runtime-semantics.md), and local event/checkpoint behavior is fixed by [persistence-semantics.md](persistence-semantics.md). Deterministic primitive behavior is fixed by [primitives-semantics.md](primitives-semantics.md). +Scheduler-integrated continuation for one immutable DAG is fixed by +[durable-recovery-semantics.md](durable-recovery-semantics.md). ## Canonical serialization v1alpha1 diff --git a/spec/conformance/durable-json.case.json b/spec/conformance/durable-json.case.json new file mode 100644 index 0000000..0a9c065 --- /dev/null +++ b/spec/conformance/durable-json.case.json @@ -0,0 +1,233 @@ +{ + "contractVersion": "graphengineering.reacher-z.github.io/durable-json-conformance/v1alpha1", + "sourceKinds": { + "json": "The value is supplied directly through the runtime portable-JSON boundary.", + "float64Bits": "The value is constructed from the exact IEEE-754 binary64 big-endian bit pattern before encoding." + }, + "validCases": [ + { + "name": "null", + "source": { "kind": "json", "value": null }, + "expect": { + "encoding": ["n"], + "canonicalJson": "[\"n\"]", + "sha256": "e00e71c16c36d75ab3d0f00c3647ac15d0f62b1fd6bec81dce7c7b5b3cc22d91" + } + }, + { + "name": "boolean-false", + "source": { "kind": "json", "value": false }, + "expect": { + "encoding": ["b", false], + "canonicalJson": "[\"b\",false]", + "sha256": "0e18bc49d2784c233215706d4ba8f6d2f39ed2df682db7ae0aa0c6d828652fdf" + } + }, + { + "name": "boolean-true", + "source": { "kind": "json", "value": true }, + "expect": { + "encoding": ["b", true], + "canonicalJson": "[\"b\",true]", + "sha256": "0dbefb4639551bc45eb8fd259a34cf1ca856f1fca6ae929eb0f876c4afc5831a" + } + }, + { + "name": "unicode-string", + "source": { "kind": "json", "value": "研究🚀" }, + "expect": { + "encoding": ["s", "研究🚀"], + "canonicalJson": "[\"s\",\"研究🚀\"]", + "sha256": "9de41f886dd112ad06f21fcbb129392ed4da88bfab57239c667095c362eb041d" + } + }, + { + "name": "safe-integer-zero", + "source": { "kind": "json", "value": 0 }, + "expect": { + "encoding": ["i", 0], + "canonicalJson": "[\"i\",0]", + "sha256": "cf90071f7873eb01c9442fcc709f5ac1106731c3a9e766e000753f2a35073c64" + } + }, + { + "name": "safe-integer-maximum", + "source": { "kind": "json", "value": 9007199254740991 }, + "expect": { + "encoding": ["i", 9007199254740991], + "canonicalJson": "[\"i\",9007199254740991]", + "sha256": "559391a8de45a57d0d2f4279a14fec1192264ca6c5e640b67f6681aa20b0996c" + } + }, + { + "name": "safe-integer-minimum", + "source": { "kind": "json", "value": -9007199254740991 }, + "expect": { + "encoding": ["i", -9007199254740991], + "canonicalJson": "[\"i\",-9007199254740991]", + "sha256": "82de17a167919d3df57af77d4871534d73ed7866c27a2ba2646621f82685b14d" + } + }, + { + "name": "float-one-point-five", + "source": { "kind": "float64Bits", "bits": "3ff8000000000000" }, + "expect": { + "encoding": ["f", "3ff8000000000000"], + "canonicalJson": "[\"f\",\"3ff8000000000000\"]", + "sha256": "402d08cb036936e5c214e5f0a14288cf45d6935aa9d798a671f9526c1a10c954" + } + }, + { + "name": "float-zero-point-one", + "source": { "kind": "float64Bits", "bits": "3fb999999999999a" }, + "expect": { + "encoding": ["f", "3fb999999999999a"], + "canonicalJson": "[\"f\",\"3fb999999999999a\"]", + "sha256": "2184d4b652ef5806e5481327c29fa93a19e30c16d1abf39dfc3647d173da51a5" + } + }, + { + "name": "float-minimum-positive-subnormal", + "source": { "kind": "float64Bits", "bits": "0000000000000001" }, + "expect": { + "encoding": ["f", "0000000000000001"], + "canonicalJson": "[\"f\",\"0000000000000001\"]", + "sha256": "c016ee244b58fae0f74ef3de6746063dac0819c909f443277d92472a7a633959" + } + }, + { + "name": "negative-zero-normalizes-to-integer-zero", + "source": { "kind": "float64Bits", "bits": "8000000000000000" }, + "expect": { + "encoding": ["i", 0], + "canonicalJson": "[\"i\",0]", + "sha256": "cf90071f7873eb01c9442fcc709f5ac1106731c3a9e766e000753f2a35073c64" + } + }, + { + "name": "integer-valued-double-normalizes-to-integer", + "source": { "kind": "float64Bits", "bits": "3ff0000000000000" }, + "expect": { + "encoding": ["i", 1], + "canonicalJson": "[\"i\",1]", + "sha256": "f9040b6e372a701015fda86238d49df633cd2e695f8446a8a480b5107ff16141" + } + }, + { + "name": "unicode-code-point-object-order", + "source": { + "kind": "json", + "value": { + "𐀀": "astral", + "\ue000": "bmp-private-use", + "a": 1 + } + }, + "expect": { + "encoding": [ + "o", + [ + ["a", ["i", 1]], + ["\ue000", ["s", "bmp-private-use"]], + ["𐀀", ["s", "astral"]] + ] + ], + "canonicalJson": "[\"o\",[[\"a\",[\"i\",1]],[\"\ue000\",[\"s\",\"bmp-private-use\"]],[\"𐀀\",[\"s\",\"astral\"]]]]", + "sha256": "9781e710d5ffac2e06b9483eaeb5f7f4414b78629f8f08d4e7ede8472d5509c3" + } + }, + { + "name": "nested-array-object", + "source": { + "kind": "json", + "value": { + "z": [null, true, { "b": "text", "a": 1 }], + "a": { "nested": [1, false] } + } + }, + "expect": { + "encoding": [ + "o", + [ + ["a", ["o", [["nested", ["a", [["i", 1], ["b", false]]]]]]], + [ + "z", + [ + "a", + [ + ["n"], + ["b", true], + ["o", [["a", ["i", 1]], ["b", ["s", "text"]]]] + ] + ] + ] + ] + ], + "canonicalJson": "[\"o\",[[\"a\",[\"o\",[[\"nested\",[\"a\",[[\"i\",1],[\"b\",false]]]]]]],[\"z\",[\"a\",[[\"n\"],[\"b\",true],[\"o\",[[\"a\",[\"i\",1]],[\"b\",[\"s\",\"text\"]]]]]]]]]", + "sha256": "e4fa829e271e60d7ab0ed53abef034ce87f7e43848c321c93829c2652b138030" + } + } + ], + "invalidEncodedCases": [ + { + "name": "unknown-tag", + "encoding": ["x"], + "reason": "The tag is not part of Durable JSON v1alpha1.", + "expect": { "schemaValid": false, "codecValid": false } + }, + { + "name": "wrong-arity", + "encoding": ["n", null], + "reason": "The n form has exactly one item.", + "expect": { "schemaValid": false, "codecValid": false } + }, + { + "name": "unsafe-integer", + "encoding": ["i", 9007199254740992], + "reason": "Integer payloads cannot exceed the interoperable safe range.", + "expect": { "schemaValid": false, "codecValid": false } + }, + { + "name": "positive-infinity-bits", + "encoding": ["f", "7ff0000000000000"], + "reason": "The f bits decode to a non-finite value.", + "expect": { "schemaValid": true, "codecValid": false } + }, + { + "name": "quiet-nan-bits", + "encoding": ["f", "7ff8000000000000"], + "reason": "The f bits decode to a non-finite value.", + "expect": { "schemaValid": true, "codecValid": false } + }, + { + "name": "integer-valued-float-bits", + "encoding": ["f", "3ff0000000000000"], + "reason": "The value 1.0 has the canonical i encoding instead.", + "expect": { "schemaValid": true, "codecValid": false } + }, + { + "name": "negative-zero-float-bits", + "encoding": ["f", "8000000000000000"], + "reason": "Negative zero normalizes to the canonical i zero encoding.", + "expect": { "schemaValid": true, "codecValid": false } + }, + { + "name": "uppercase-float-bits", + "encoding": ["f", "3FF8000000000000"], + "reason": "The binary64 payload must use exactly sixteen lowercase hexadecimal digits.", + "expect": { "schemaValid": false, "codecValid": false } + }, + { + "name": "unsorted-object-keys", + "encoding": ["o", [["b", ["n"]], ["a", ["n"]]]], + "reason": "Object keys must be strictly increasing by Unicode code point.", + "expect": { "schemaValid": true, "codecValid": false } + }, + { + "name": "duplicate-object-keys", + "encoding": ["o", [["a", ["n"]], ["a", ["b", true]]]], + "reason": "Strict key ordering forbids duplicate normalized keys.", + "expect": { "schemaValid": true, "codecValid": false } + } + ] +} diff --git a/spec/conformance/durable-resume.case.json b/spec/conformance/durable-resume.case.json new file mode 100644 index 0000000..2b9abc5 --- /dev/null +++ b/spec/conformance/durable-resume.case.json @@ -0,0 +1,94 @@ +{ + "caseVersion": "scheduler-recovery/v1alpha1", + "name": "resume reuses committed success and advances an interrupted safe node", + "runId": "resume-basic", + "implementationId": "fixture@1", + "graph": { + "apiVersion": "graphengineering.reacher-z.github.io/v1alpha1", + "kind": "Graph", + "metadata": { "name": "durable-resume", "version": "1" }, + "inputSchema": {}, + "outputSchema": {}, + "entrypoints": ["left", "right"], + "outputs": { "result": { "node": "merge" } }, + "nodes": [ + { + "id": "left", + "kind": "agent", + "inputSchema": {}, + "outputSchema": {}, + "config": {}, + "retry": { "maxAttempts": 2 }, + "sideEffects": "none" + }, + { + "id": "right", + "kind": "agent", + "inputSchema": {}, + "outputSchema": {}, + "config": {}, + "retry": { "maxAttempts": 2 }, + "sideEffects": "none" + }, + { + "id": "merge", + "kind": "agent", + "inputSchema": {}, + "outputSchema": {}, + "config": {}, + "sideEffects": "none" + } + ], + "edges": [ + { + "id": "left-merge", + "from": { "node": "left" }, + "to": { "node": "merge", "port": "left" } + }, + { + "id": "right-merge", + "from": { "node": "right" }, + "to": { "node": "merge", "port": "right" } + } + ], + "policies": { "maxConcurrency": 2, "maxTotalAttempts": 4 } + }, + "graphInput": { "seed": 1 }, + "preCrash": { + "succeeded": { + "nodeId": "left", + "attempt": 1, + "input": { "seed": 1 }, + "output": { "left": true } + }, + "interrupted": { + "nodeId": "right", + "attempt": 1, + "input": { "seed": 1 } + } + }, + "resumeReturns": { + "right": { "right": true }, + "merge": { "merged": true } + }, + "expect": { + "executorCalls": [ + { "nodeId": "right", "attempt": 2 }, + { "nodeId": "merge", "attempt": 1 } + ], + "neverExecute": ["left"], + "reusedNodeIds": ["left"], + "interruptedNodeIds": ["right"], + "totalAttempts": 4, + "status": "succeeded", + "output": { "result": { "merged": true } }, + "mergeInput": { + "left": { "left": true }, + "right": { "right": true } + }, + "terminalResume": { + "newEvents": 0, + "executorCalls": 0 + } + } +} diff --git a/spec/conformance/expected.json b/spec/conformance/expected.json index a3e7700..be52db6 100644 --- a/spec/conformance/expected.json +++ b/spec/conformance/expected.json @@ -34,6 +34,14 @@ "valid": false, "diagnosticCodes": ["GE1006_UNREACHABLE_NODE"] }, + "invalid-unsafe-budget.graph.json": { + "valid": false, + "diagnosticCodes": ["GE1007_INVALID_GRAPH"] + }, + "invalid-oversized-timers.graph.json": { + "valid": false, + "diagnosticCodes": ["GE1007_INVALID_GRAPH"] + }, "invalid-entrypoint-incoming.graph.json": { "valid": false, "diagnosticCodes": ["GE1010_ENTRYPOINT_HAS_INCOMING", "GE1006_UNREACHABLE_NODE"] diff --git a/spec/conformance/invalid-oversized-timers.graph.json b/spec/conformance/invalid-oversized-timers.graph.json new file mode 100644 index 0000000..9d8a9da --- /dev/null +++ b/spec/conformance/invalid-oversized-timers.graph.json @@ -0,0 +1,26 @@ +{ + "apiVersion": "graphengineering.reacher-z.github.io/v1alpha1", + "kind": "Graph", + "metadata": { "name": "oversized-timers", "version": "1" }, + "inputSchema": {}, + "outputSchema": {}, + "entrypoints": ["root"], + "outputs": { "result": { "node": "root" } }, + "nodes": [ + { + "id": "root", + "kind": "transform", + "inputSchema": {}, + "outputSchema": {}, + "config": {}, + "retry": { + "maxAttempts": 2, + "initialDelayMs": 2147483648, + "maxDelayMs": 2147483648 + }, + "timeoutMs": 2147483648 + } + ], + "edges": [], + "policies": { "maxDurationMs": 2147483648 } +} diff --git a/spec/conformance/invalid-unsafe-budget.graph.json b/spec/conformance/invalid-unsafe-budget.graph.json new file mode 100644 index 0000000..857e129 --- /dev/null +++ b/spec/conformance/invalid-unsafe-budget.graph.json @@ -0,0 +1,20 @@ +{ + "apiVersion": "graphengineering.reacher-z.github.io/v1alpha1", + "kind": "Graph", + "metadata": { "name": "unsafe-budget", "version": "1" }, + "inputSchema": {}, + "outputSchema": {}, + "entrypoints": ["root"], + "outputs": { "result": { "node": "root" } }, + "nodes": [ + { + "id": "root", + "kind": "transform", + "inputSchema": {}, + "outputSchema": {}, + "config": {} + } + ], + "edges": [], + "policies": { "maxTotalAttempts": 9007199254740992 } +} diff --git a/spec/conformance/strict-rfc3339.case.json b/spec/conformance/strict-rfc3339.case.json new file mode 100644 index 0000000..d3e60de --- /dev/null +++ b/spec/conformance/strict-rfc3339.case.json @@ -0,0 +1,30 @@ +{ + "caseVersion": "strict-rfc3339/v1alpha1", + "valid": [ + "0001-01-01T00:00:00Z", + "2000-02-29T23:59:59.999999Z", + "2024-02-29T12:34:56+00:00", + "2026-07-26T12:34:56.125-07:00", + "9999-12-31T23:59:59+23:59" + ], + "invalid": [ + "0000-01-01T00:00:00Z", + "1900-02-29T00:00:00Z", + "2023-02-29T00:00:00Z", + "2024-02-30T00:00:00Z", + "2026-00-01T00:00:00Z", + "2026-04-31T00:00:00Z", + "2026-13-01T00:00:00Z", + "2026-01-00T00:00:00Z", + "2026-01-32T00:00:00Z", + "2026-01-01T24:00:00Z", + "2026-01-01T00:60:00Z", + "2026-01-01T00:00:60Z", + "2026-01-01T00:00:00+24:00", + "2026-01-01T00:00:00+00:60", + "2026-01-01T00:00:00", + "2026-01-01", + "20260101T000000Z", + "not-a-date" + ] +} diff --git a/spec/durable-json.schema.json b/spec/durable-json.schema.json new file mode 100644 index 0000000..4c7b5a7 --- /dev/null +++ b/spec/durable-json.schema.json @@ -0,0 +1,103 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://reacher-z.github.io/GraphEngineering/schemas/v1alpha1/durable-json.schema.json", + "title": "Graph Engineering Tagged Durable JSON", + "description": "Canonical, checkpoint-safe representation of portable finite JSON values. Semantic validation must additionally reject non-finite or integer-valued f payloads and object entries whose keys are not strictly increasing by Unicode code point.", + "$ref": "#/$defs/value", + "$defs": { + "value": { + "oneOf": [ + { "$ref": "#/$defs/nullValue" }, + { "$ref": "#/$defs/booleanValue" }, + { "$ref": "#/$defs/stringValue" }, + { "$ref": "#/$defs/integerValue" }, + { "$ref": "#/$defs/floatValue" }, + { "$ref": "#/$defs/arrayValue" }, + { "$ref": "#/$defs/objectValue" } + ] + }, + "nullValue": { + "type": "array", + "prefixItems": [{ "const": "n" }], + "items": false, + "minItems": 1, + "maxItems": 1 + }, + "booleanValue": { + "type": "array", + "prefixItems": [{ "const": "b" }, { "type": "boolean" }], + "items": false, + "minItems": 2, + "maxItems": 2 + }, + "stringValue": { + "type": "array", + "prefixItems": [{ "const": "s" }, { "type": "string" }], + "items": false, + "minItems": 2, + "maxItems": 2 + }, + "integerValue": { + "type": "array", + "prefixItems": [ + { "const": "i" }, + { + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 + } + ], + "items": false, + "minItems": 2, + "maxItems": 2 + }, + "floatValue": { + "type": "array", + "description": "The payload is an IEEE-754 binary64 bit pattern in big-endian order. Canonical semantic validation rejects NaN, infinities, negative zero, and every value representable by the i form.", + "prefixItems": [ + { "const": "f" }, + { "type": "string", "pattern": "^[0-9a-f]{16}$" } + ], + "items": false, + "minItems": 2, + "maxItems": 2 + }, + "arrayValue": { + "type": "array", + "prefixItems": [ + { "const": "a" }, + { + "type": "array", + "items": { "$ref": "#/$defs/value" } + } + ], + "items": false, + "minItems": 2, + "maxItems": 2 + }, + "objectEntry": { + "type": "array", + "prefixItems": [ + { "type": "string" }, + { "$ref": "#/$defs/value" } + ], + "items": false, + "minItems": 2, + "maxItems": 2 + }, + "objectValue": { + "type": "array", + "description": "Entries must be strictly increasing by Unicode code point. Strict ordering makes duplicate keys invalid; JSON Schema validates the shape while the codec enforces this semantic invariant.", + "prefixItems": [ + { "const": "o" }, + { + "type": "array", + "items": { "$ref": "#/$defs/objectEntry" } + } + ], + "items": false, + "minItems": 2, + "maxItems": 2 + } + } +} diff --git a/spec/durable-recovery-semantics.md b/spec/durable-recovery-semantics.md new file mode 100644 index 0000000..636b11f --- /dev/null +++ b/spec/durable-recovery-semantics.md @@ -0,0 +1,376 @@ +# Durable scheduler recovery semantics v1alpha1 + +This document freezes the first scheduler-integrated recovery contract shared +by the TypeScript and Python runtimes. It covers continuation of one immutable, +compiled DAG after process loss. It does not define replay, fork, dynamic graph +patches, explicit cycles, streaming edges, distributed leases, or exactly-once +external effects. + +The append-only `EventStore` is the source of truth. A checkpoint is only a +rebuildable projection cache and cannot authorize work by itself. + +## Public operations + +Each runtime exposes separate start and resume operations. Start requires a +graph, graph input, a safe `runId`, a non-empty caller-supplied +`implementationId`, an event store, and executors. Resume requires the same +graph, `runId`, `implementationId`, event store, and executors, but reads the +original graph input from `RunCreated`. + +- Start fails with `RUN_ALREADY_EXISTS` when the stream is non-empty. +- Resume fails with `RUN_NOT_FOUND` when the stream is empty. +- Start never silently resumes, and resume never silently creates a run. +- The existing in-memory scheduler API and behavior remain unchanged. +- Durable operational errors are typed, structured errors with a stable code, + `runId`, message, and optional details. Graph execution failures remain normal + structured graph results and terminal events. + +`implementationId` is a caller assertion about the executor set. The runtime +stores only its tagged Durable JSON SHA-256 as `implementationHash`. This prevents an +accidental resume under a differently declared implementation; it cannot prove +that a caller labeled changed code honestly. + +## Bound identity + +`RunCreated` binds the run to all of the following: + +- `graphRevision`, fixed to `1` in this contract; +- the compiler's canonical `graphHash`; +- `inputHash`, the Durable JSON SHA-256 of the detached graph input; +- `implementationHash`, the Durable JSON SHA-256 of `implementationId`; +- the detached original input itself; +- the effective `maxTotalAttempts` value. + +Resume recompiles a detached graph snapshot and validates every bound value +before it appends an event or invokes an executor. A mismatch produces +`GRAPH_HASH_MISMATCH`, `INPUT_HASH_MISMATCH`, or `IMPLEMENTATION_MISMATCH` as +applicable. Because resume reads the original input from history, its public API +does not accept replacement input. + +The durable payload boundary accepts the same portable finite JSON values as the +runtime. To preserve every finite binary64 value without relying on +language-specific decimal rendering, persisted inputs, outputs, result +snapshots, and their hashes use the tagged Durable JSON encoding below. NaN, +infinities, unsafe integers, sparse arrays, non-JSON objects, and cycles remain +invalid. + +## Tagged Durable JSON + +The encoding is a JSON array whose first item is a one-character tag: + +```text +null => ["n"] +boolean => ["b", true] +string => ["s", "text"] +safe integer => ["i", 42] +finite non-integer => ["f", "3ff8000000000000"] +array => ["a", [, ...]] +object => ["o", [["key", ], ...]] +``` + +The `f` payload is exactly sixteen lowercase hexadecimal digits containing the +IEEE-754 binary64 bits in big-endian byte order. Object entries are sorted by +Unicode code point and duplicate normalized keys are invalid. Integer-valued +doubles and negative zero normalize to their safe-integer form, matching the +runtime JSON boundary. Decoders reject unknown tags, wrong arity, unsorted or +duplicate object keys, non-safe `i` payloads, and `f` payloads that decode to a +non-finite or integer-valued value. + +`inputHash`, `outputHash`, implementation hashes, activity keys, and result +hashes are lowercase SHA-256 of canonical UTF-8 JSON for the tagged value. The +tagged representation itself contains no floating-point number and can be +stored directly inside a v1alpha1 checkpoint. + +## Event integrity and stream rules + +Every scheduler-written event conforms to `event.schema.json` and has: + +- `graphRevision: 1`; +- the run's exact `runId`; +- the next contiguous sequence; +- `payloadHash` equal to lowercase SHA-256 of canonical JSON for `data`; +- a non-empty event ID that is unique within the run and a strict, real-calendar + RFC 3339 timestamp. Years are `0001` through `9999`; year `0000`, impossible + calendar dates, leap seconds, and offsets outside `00:00` through `23:59` + are invalid. + +The default event ID may be deterministic from `runId` and sequence. Clock and +event-ID factories are injectable for tests. Stored time and IDs are facts once +appended and are never regenerated during resume. + +The runtime semantically validates the complete event history after the store's +envelope/sequence validation. At minimum it rejects: + +- a missing, duplicate, misplaced, or incompatible `RunCreated`; +- duplicate event IDs, including a collision with an event already in history; +- lifecycle events before `RunStarted`; +- an event after a terminal event; +- a duplicate terminal event; +- unknown nodes or edges; +- non-contiguous or regressing attempts for one node; +- a start, outcome, or retry without its required predecessor; +- more than one outcome for one attempt; +- a success after a node already settled; +- an incorrect payload hash, graph revision, input hash, output hash, activity + key, or result payload; +- a terminal result inconsistent with the folded history. + +Semantic violations fail with `INVALID_RUN_HISTORY`. Corrupt bytes and invalid +event envelopes retain the persistence layer's structured corruption error. + +The writer applies the same requirements before append. Failure of an injected +clock or event-ID factory, an invalid generated timestamp or ID, and an event +construction error are durability failures. They latch `DURABILITY_STORE_FAILED` +and stop new scheduling just like an event-store append failure; they are never +returned as raw host-language exceptions while sibling work continues writing. + +## Event payloads + +`RunCreated.data` has this exact shape: + +```json +{ + "contractVersion": "scheduler-recovery/v1alpha1", + "graphHash": "<64 lowercase hex>", + "implementationHash": "<64 lowercase hex>", + "input": ["o", []], + "inputHash": "<64 lowercase hex>", + "maxTotalAttempts": 8 +} +``` + +`RunStarted.data` is an empty object. `RunResumed.data` contains node IDs in +graph declaration order: + +```json +{ + "reusedNodeIds": ["left"], + "interruptedNodeIds": ["right"] +} +``` + +`NodeScheduled.data` contains the detached bound input and recovery identity: + +```json +{ + "input": ["o", [["seed", ["i", 1]]]], + "inputHash": "<64 lowercase hex>", + "activityKey": "<64 lowercase hex>", + "sideEffects": "none" +} +``` + +`NodeStarted.data` contains `inputHash` and `activityKey`. A normal scheduler +append writes `NodeScheduled` immediately followed by `NodeStarted` in one CAS +batch. A durable `NodeStarted` is the attempt claim: it consumes both the node +retry budget and the global attempt budget before executor code runs. + +The stable logical activity key is: + +```text +durableJsonHash([ + "activity/v1alpha1", runId, graphRevision, nodeId, inputHash +]) +``` + +Attempt is deliberately excluded. Executors receive `runId`, `attemptId`, and +`idempotencyKey`/`activityKey` in their durable execution context. + +`NodeSucceeded.data` contains `inputHash`, the tagged detached validated output, and its +canonical `outputHash`. The runtime appends `NodeSucceeded` and every outgoing +`EdgeEmitted` in one CAS batch; edge events are ordered by edge ID using Unicode +code-point order. Each `EdgeEmitted.data` contains the producer `outputHash`. + +`NodeAttemptFailed.data` has this shape: + +```json +{ + "terminal": false, + "failure": { + "phase": "execute", + "code": "NODE_EXECUTION_INTERRUPTED", + "message": "process ended before the attempt outcome was durably recorded", + "nodeId": "right", + "attempt": 1, + "retryable": true, + "causeName": "ProcessLost" + } +} +``` + +Some scheduler outcomes settle a node without starting a new attempt: missing +executor configuration, deterministic input-binding failure, upstream failure, +attempt-budget exhaustion, or cancellation before dispatch. They are not left +as unaudited fields inside the terminal snapshot. The scheduler commits a +`NodeSettledWithoutAttempt` event before releasing dependants. Its data object +has exactly one `result` field containing Tagged Durable JSON. The decoded value +is the same exact node-result object used in the terminal result: `nodeId`, topological +`sequence`, `status`, `attempts`, optional bound `input`, and a structured +`failure`. Its status must be `failed` or `skipped`; it cannot contain an +`output`; its attempt count and failure identity must equal the folded history. +Input presence is significant: an absent `input` means input binding never +completed, while an explicitly present tagged null is a successfully bound JSON +null input. +The envelope requires `nodeId` and omits `attempt`. The fold treats this event as +both the node's first scheduling fact when no `NodeScheduled` exists and its +completion fact. Consequently every terminal node result is derived from an +explicit node outcome event rather than trusted only because it appears in the +terminal snapshot. + +When another attempt is allowed, the same CAS batch appends `NodeRetried` after +`NodeAttemptFailed`. `NodeRetried.attempt` names the next attempt, and its data +contains the absolute RFC 3339 `availableAt` time and `activityKey`. The retry +event does not consume a budget; the next `NodeStarted` does. Resume waits only +the remaining delay, rather than restarting the whole backoff. + +A retry reservation is exclusive. Once `NodeRetried` reserves attempt `N`, a +late `NodeStarted` for an older attempt cannot reopen the node, and a second +`NodeStarted` cannot follow a recorded outcome. Pending retry reservations count +against the global attempt budget even before their corresponding starts. + +The terminal `RunSucceeded`, `RunFailed`, or `RunCancelled` event stores a +tagged portable `result` snapshot in `data`. This makes terminal resume idempotent and +allows it to return the recorded result without rerunning scheduling logic. +Every graph node must have an explicit folded outcome and no retry reservation +may remain pending before a terminal event is valid. + +Structured failure identity is cross-language data; human diagnostic prose is +not. Consumers validate deterministic fields such as phase, code, node ID, +attempt, retryability, output name, and port when applicable, but preserve and +accept a producer's non-empty `message` and optional host-specific cause name. +The synthetic `NODE_EXECUTION_INTERRUPTED` message and `ProcessLost` cause are +the exception: both are canonical facts defined above. An output-binding +failure has the portable fields `phase`, `code`, `message`, `outputName`, +`nodeId`, and optional `port`; it does not acquire attempt or retryability fields. + +## Commit-before-release invariant + +For every attempt, persistence and execution are ordered as follows: + +```text +NodeScheduled + NodeStarted committed + -> executor may run + -> output validates and detaches + -> NodeSucceeded + EdgeEmitted committed + -> dependants may become ready +``` + +For a node that settles without dispatch, the corresponding invariant is +`NodeSettledWithoutAttempt committed -> dependants may become ready`. + +An append failure is never converted to a successful node. A dependent cannot +start, be scheduled, or observe an output until the success batch has returned +successfully. Parallel append calls for one run are serialized locally, and each +uses the last successful sequence as its expected version. + +A compare-and-swap conflict stops new scheduling and surfaces +`RESUME_CONFLICT`; it is never blindly retried. Executors already in flight may +still have produced external effects, which is one reason CAS is not a lease. + +## Recovery projection + +Resume reads and folds the complete event stream before claiming continuation. +The fold reconstructs: + +- original graph input; +- successful node inputs and outputs; +- final node failures; +- claimed attempts per node and globally; +- retry availability times; +- open `NodeStarted` attempts; +- run terminal state and recorded result. + +It then appends `RunResumed` with expected-version CAS before executor code is +allowed to run. A losing resume returns `RESUME_CONFLICT` and invokes no +executor. The scheduler seeds committed successful results into its normal +ready-queue engine. Those nodes are never executed again, and their dependants +consume their recorded outputs. + +An open `NodeStarted` has an unknown outcome. Resume records a synthetic +`NODE_EXECUTION_INTERRUPTED` outcome before deciding whether to continue: + +- `sideEffects: "none"` may retry automatically; +- `sideEffects: "idempotent"` may retry with the same activity key; the + application is responsible for actually passing that key to the external + system; +- `sideEffects: "non-idempotent"` or an omitted declaration may not retry. + This contract fails closed with `IN_DOUBT_SIDE_EFFECT`. An auditable + application approval/reconciliation protocol is intentionally outside this + v1alpha1 slice and must not be simulated by silently reinvoking the executor. + +The interrupted attempt remains consumed. Its successor uses attempt `N + 1`. +If either the per-node or global attempt budget is exhausted, the node settles +as failed without invoking an executor. + +Cancellation signals do not survive a process. A committed `RunCancelled` is +terminal. A crash without a terminal cancellation event is recovered from its +durable node history under the same unknown-outcome rules. + +## Terminal idempotence + +When a valid terminal event already exists, resume: + +- returns its recorded graph result; +- appends no event; +- saves no checkpoint; +- invokes no executor; +- reports every node/output/failure exactly as recorded. + +## Checkpoints + +Scheduler recovery in this revision is correct from events alone. Checkpoint +integration is optional acceleration and must not weaken that behavior. + +When implemented, a scheduler checkpoint must contain the last applied event +sequence, graph/input/implementation hashes, a history-prefix hash, total +attempts, and node projections in graph declaration order. Inputs and outputs +use tagged Durable JSON so the checkpoint's safe-integer-only state remains +valid even when runtime values contain finite decimals. + +A checkpoint must never lead the event stream. Missing, stale, corrupt, +ahead-of-tail, or projection-inconsistent checkpoints are ignored with a +structured recovery warning and rebuilt from events. Event corruption is never +hidden by a checkpoint. Checkpoint writes occur only after authoritative event +commit and before a dependent is released if the configured policy requires a +fresh cache. This document does not make checkpoint availability a correctness +requirement. + +## Stable durable error codes + +- `RUN_NOT_FOUND` +- `RUN_ALREADY_EXISTS` +- `GRAPH_HASH_MISMATCH` +- `INPUT_HASH_MISMATCH` +- `IMPLEMENTATION_MISMATCH` +- `INVALID_RUN_HISTORY` +- `NODE_EXECUTION_INTERRUPTED` +- `IN_DOUBT_SIDE_EFFECT` +- `RESUME_CONFLICT` +- `DURABILITY_STORE_FAILED` + +Persistence codes such as `UNSAFE_IDENTIFIER`, `VERSION_CONFLICT`, +`CORRUPT_EVENT_LOG`, and `PERSISTENCE_IO` remain available as causal detail. + +## Explicit limits + +This contract supports continuation only for one static revision-1 DAG. The +local JSONL store is documented for one coordinating process and private local +storage. CAS detects stale writes; it does not prevent two processes from +executing external work before one loses a write race. Distributed resume needs +a real lease/fencing provider. + +External effects are at-least-once. A crash can occur after the outside system +commits and before `NodeSucceeded` commits. Stable activity keys, idempotent +remote APIs, reconciliation, compensation, and human approval are application +responsibilities. This revision does not expose an approval callback or record +an approval decision. The project does not claim universal exactly-once +execution. + +All scheduler durations that reach a host timer are bounded to a signed 32-bit +millisecond interval. `initialDelayMs` and `maxDelayMs` are in +`0..2147483647`; `timeoutMs` and `maxDurationMs` are in `1..2147483647`. +Larger values fail graph compilation with `GE1007_INVALID_GRAPH`. Count budgets +remain independent safe integers subject to their existing field-specific +limits. Date arithmetic or host timer failures after compilation latch +`DURABILITY_STORE_FAILED` instead of silently overflowing, wrapping, or running +immediately. diff --git a/spec/event.schema.json b/spec/event.schema.json index 0dfb81e..673f28e 100644 --- a/spec/event.schema.json +++ b/spec/event.schema.json @@ -12,7 +12,8 @@ "enum": [ "RunCreated", "RunStarted", "RunPaused", "RunResumed", "GraphPatched", "NodeScheduled", "NodeStarted", "NodeAttemptFailed", - "NodeRetried", "NodeSucceeded", "EdgeEmitted", "BarrierSatisfied", + "NodeSettledWithoutAttempt", "NodeRetried", "NodeSucceeded", + "EdgeEmitted", "BarrierSatisfied", "RouteSelected", "VerificationRecorded", "BudgetUpdated", "ArtifactCreated", "HumanInputRequested", "HumanInputReceived", "RunCancelled", "RunFailed", "RunSucceeded" diff --git a/spec/graph.schema.json b/spec/graph.schema.json index 89aab56..2a9bec7 100644 --- a/spec/graph.schema.json +++ b/spec/graph.schema.json @@ -63,8 +63,8 @@ "additionalProperties": false, "properties": { "maxAttempts": { "type": "integer", "minimum": 1, "maximum": 100 }, - "initialDelayMs": { "type": "integer", "minimum": 0 }, - "maxDelayMs": { "type": "integer", "minimum": 0 }, + "initialDelayMs": { "type": "integer", "minimum": 0, "maximum": 2147483647 }, + "maxDelayMs": { "type": "integer", "minimum": 0, "maximum": 2147483647 }, "backoffMultiplier": { "type": "number", "minimum": 1 }, "jitter": { "type": "boolean" } } @@ -82,7 +82,7 @@ "outputSchema": { "$ref": "#/$defs/jsonSchema" }, "config": {}, "retry": { "$ref": "#/$defs/retry" }, - "timeoutMs": { "type": "integer", "minimum": 1 }, + "timeoutMs": { "type": "integer", "minimum": 1, "maximum": 2147483647 }, "cache": { "type": "object" }, "resources": { "type": "object" }, "isolation": { "type": "object" }, @@ -107,12 +107,12 @@ "type": "object", "additionalProperties": true, "properties": { - "maxConcurrency": { "type": "integer", "minimum": 1 }, - "maxDynamicNodes": { "type": "integer", "minimum": 0 }, - "maxDepth": { "type": "integer", "minimum": 1 }, - "maxFanOut": { "type": "integer", "minimum": 1 }, - "maxTotalAttempts": { "type": "integer", "minimum": 1 }, - "maxDurationMs": { "type": "integer", "minimum": 1 }, + "maxConcurrency": { "type": "integer", "minimum": 1, "maximum": 9007199254740991 }, + "maxDynamicNodes": { "type": "integer", "minimum": 0, "maximum": 9007199254740991 }, + "maxDepth": { "type": "integer", "minimum": 1, "maximum": 9007199254740991 }, + "maxFanOut": { "type": "integer", "minimum": 1, "maximum": 9007199254740991 }, + "maxTotalAttempts": { "type": "integer", "minimum": 1, "maximum": 9007199254740991 }, + "maxDurationMs": { "type": "integer", "minimum": 1, "maximum": 2147483647 }, "maxCostUsd": { "type": "number", "minimum": 0 } } } diff --git a/spec/persistence-semantics.md b/spec/persistence-semantics.md index a256305..c62d270 100644 --- a/spec/persistence-semantics.md +++ b/spec/persistence-semantics.md @@ -1,9 +1,9 @@ # Persistence semantics v1alpha1 This document freezes the observable contract shared by the TypeScript and -Python local persistence implementations. It describes storage primitives, not -durable scheduler recovery. Resume, replay, leases, and effectively-once node -commit remain later runtime work until their own conformance cases exist. +Python local persistence implementations. Scheduler integration is specified +separately in `durable-recovery-semantics.md`; replay, fork, and distributed +leases remain later runtime work. ## Event stream versioning diff --git a/spec/runtime-semantics.md b/spec/runtime-semantics.md index b064730..1502952 100644 --- a/spec/runtime-semantics.md +++ b/spec/runtime-semantics.md @@ -77,6 +77,8 @@ Events receive a monotonically increasing sequence per run. Appends use an expected prior sequence/version. A validated node result is persisted before dependents become schedulable. Resume reconstructs state from durable history and cannot rerun a successful node unless a new replay/fork history requests it. +The executable continuation contract, event payloads, crash windows, and +side-effect safety gates are defined by `durable-recovery-semantics.md`. ## Determinism boundary diff --git a/tools/conformance/python_durable_interop.py b/tools/conformance/python_durable_interop.py new file mode 100644 index 0000000..3d3005f --- /dev/null +++ b/tools/conformance/python_durable_interop.py @@ -0,0 +1,145 @@ +"""Produce and consume terminal durable histories across the Python/TS boundary.""" + +from __future__ import annotations + +import asyncio +import json +import sys +from typing import Any + +from graph_engineering import ( + GraphEvent, + NodeContext, + RunResult, + compile_graph, + decode_durable_json, + resume_graph_run, + start_graph_run, +) +from graph_engineering.persistence import MemoryEventStore + + +def _normalize_result(result: RunResult) -> dict[str, Any]: + return { + "status": result.status.value, + "nodeStatuses": { + node_id: node.status.value for node_id, node in result.nodes.items() + }, + "failureCodes": [failure.code.value for failure in result.failures], + "attempts": { + node_id: node.attempts for node_id, node in result.nodes.items() + }, + "outputs": dict(result.outputs or {}), + "totalAttempts": result.total_attempts, + } + + +def _serialize_events(events: tuple[GraphEvent, ...]) -> list[dict[str, Any]]: + return [ + event.model_dump(mode="json", by_alias=True, exclude_none=True) + for event in events + ] + + +def _settled_semantics(events: tuple[GraphEvent, ...]) -> dict[str, Any] | None: + settled = next( + (event for event in events if event.type == "NodeSettledWithoutAttempt"), + None, + ) + if settled is None: + return None + result = decode_durable_json(settled.data["result"]) + if not isinstance(result, dict): + raise TypeError("NodeSettledWithoutAttempt.result must decode to an object") + failure = result.get("failure") + if not isinstance(failure, dict): + raise TypeError("NodeSettledWithoutAttempt.result.failure must be an object") + return { + "nodeId": result.get("nodeId"), + "status": result.get("status"), + "attempts": result.get("attempts"), + "hasInput": "input" in result, + "input": result.get("input"), + "failureCode": failure.get("code"), + } + + +async def _produce(case: dict[str, Any]) -> dict[str, Any]: + graph = compile_graph(case["graph"]) + store = MemoryEventStore() + handlers = {} + if case["mode"] == "outputBinding": + handlers = {"root": lambda _: {"actual": True}} + elif case["mode"] != "missingExecutor": + raise AssertionError(f"unknown durable interop mode: {case['mode']!r}") + + result = await start_graph_run( + graph, + case["graphInput"], + handlers, + run_id=case["pythonRunId"], + implementation_id=case["implementationId"], + event_store=store, + clock=lambda: case["fixedTime"], + ) + events = await store.read(case["pythonRunId"]) + return { + "result": _normalize_result(result), + "eventTypes": [event.type for event in events], + "settled": _settled_semantics(events), + "events": _serialize_events(events), + } + + +async def _consume_typescript(case: dict[str, Any]) -> dict[str, Any]: + graph = compile_graph(case["graph"]) + events = tuple(GraphEvent.model_validate(event) for event in case["tsEvents"]) + store = MemoryEventStore() + await store.append(case["tsRunId"], -1, events) + before = len(events) + executor_calls = 0 + + def must_not_run(_: NodeContext) -> Any: + nonlocal executor_calls + executor_calls += 1 + raise AssertionError("terminal cross-language resume invoked an executor") + + result = await resume_graph_run( + graph, + {"root": must_not_run}, + run_id=case["tsRunId"], + implementation_id=case["implementationId"], + event_store=store, + clock=lambda: case["fixedTime"], + ) + after = await store.read(case["tsRunId"]) + return { + "result": _normalize_result(result), + "eventTypes": [event.type for event in after], + "settled": _settled_semantics(after), + "terminalResume": { + "newEvents": len(after) - before, + "executorCalls": executor_calls, + }, + } + + +async def _main() -> None: + request = json.load(sys.stdin) + cases = request.get("cases") + if not isinstance(cases, list) or not cases: + raise AssertionError("durable interop request must contain a non-empty cases array") + + report: dict[str, Any] = {} + for case in cases: + if not isinstance(case, dict) or not isinstance(case.get("name"), str): + raise TypeError("every durable interop case must be a named object") + report[case["name"]] = { + "pythonProduced": await _produce(case), + "pythonConsumedTs": await _consume_typescript(case), + } + print(json.dumps(report, ensure_ascii=False, sort_keys=True, separators=(",", ":"))) + + +if __name__ == "__main__": + asyncio.run(_main()) diff --git a/tools/conformance/python_durable_report.py b/tools/conformance/python_durable_report.py new file mode 100755 index 0000000..082883e --- /dev/null +++ b/tools/conformance/python_durable_report.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +"""Run the shared crash/resume case through the native Python runtime.""" + +from __future__ import annotations + +import asyncio +import json +from pathlib import Path +from typing import Any + +from graph_engineering import ( + NodeContext, + compile_graph, + resume_graph_run, + start_graph_run, +) +from graph_engineering.persistence import MemoryEventStore + +ROOT = Path(__file__).resolve().parents[2] +CASE_PATH = ROOT / "spec" / "conformance" / "durable-resume.case.json" +FIXED_TIME = "2026-07-26T12:00:00.000Z" + + +class ProcessLost(BaseException): + """Escape normal attempt handling to model abrupt process loss.""" + + +async def execute() -> dict[str, Any]: + case = json.loads(CASE_PATH.read_text(encoding="utf-8")) + graph = compile_graph(case["graph"]) + store = MemoryEventStore() + + async def left(_: NodeContext) -> Any: + return case["preCrash"]["succeeded"]["output"] + + async def right_crashes(_: NodeContext) -> Any: + await asyncio.sleep(0.02) + raise ProcessLost + + try: + await start_graph_run( + graph, + case["graphInput"], + {"left": left, "right": right_crashes, "merge": lambda _: None}, + run_id=case["runId"], + implementation_id=case["implementationId"], + event_store=store, + clock=lambda: FIXED_TIME, + ) + except ProcessLost: + pass + else: + raise AssertionError("the pre-crash execution unexpectedly completed") + + before_resume = await store.read(case["runId"]) + executor_calls: list[dict[str, Any]] = [] + merge_input: Any = None + resumed_activity_keys: list[str | None] = [] + + def left_must_not_run(_: NodeContext) -> Any: + raise AssertionError("resume reran the committed left node") + + def right(context: NodeContext) -> Any: + executor_calls.append({"nodeId": context.node.id, "attempt": context.attempt}) + resumed_activity_keys.append(context.activity_key) + return case["resumeReturns"]["right"] + + def merge(context: NodeContext) -> Any: + nonlocal merge_input + executor_calls.append({"nodeId": context.node.id, "attempt": context.attempt}) + merge_input = context.input + return case["resumeReturns"]["merge"] + + result = await resume_graph_run( + graph, + {"left": left_must_not_run, "right": right, "merge": merge}, + run_id=case["runId"], + implementation_id=case["implementationId"], + event_store=store, + clock=lambda: FIXED_TIME, + ) + after_resume = await store.read(case["runId"]) + resumed = next(event for event in after_resume if event.type == "RunResumed") + right_schedules = [ + event + for event in after_resume + if event.type == "NodeScheduled" and event.node_id == "right" + ] + + terminal_calls = 0 + + def terminal_must_not_run(_: NodeContext) -> Any: + nonlocal terminal_calls + terminal_calls += 1 + raise AssertionError("terminal resume invoked an executor") + + terminal_version = len(after_resume) + terminal = await resume_graph_run( + graph, + { + "left": terminal_must_not_run, + "right": terminal_must_not_run, + "merge": terminal_must_not_run, + }, + run_id=case["runId"], + implementation_id=case["implementationId"], + event_store=store, + clock=lambda: FIXED_TIME, + ) + final_history = await store.read(case["runId"]) + + return { + "status": result.status.value, + "output": dict(result.outputs or {}), + "nodeStatuses": { + node_id: node.status.value for node_id, node in result.nodes.items() + }, + "attempts": { + node_id: node.attempts for node_id, node in result.nodes.items() + }, + "totalAttempts": result.total_attempts, + "executorCalls": executor_calls, + "mergeInput": merge_input, + "reusedNodeIds": resumed.data["reusedNodeIds"], + "interruptedNodeIds": resumed.data["interruptedNodeIds"], + "preCrashEventTypes": [event.type for event in before_resume], + "preCrashEvents": [ + event.model_dump(mode="json", by_alias=True, exclude_none=True) + for event in before_resume + ], + "resumeEventTypes": [ + event.type for event in after_resume[len(before_resume) :] + ], + "activityKeyStable": ( + len(right_schedules) == 2 + and right_schedules[0].data["activityKey"] + == right_schedules[1].data["activityKey"] + and resumed_activity_keys + == [right_schedules[0].data["activityKey"]] + ), + "terminalResume": { + "newEvents": len(final_history) - terminal_version, + "executorCalls": terminal_calls, + "sameResult": terminal == result, + }, + } + + +if __name__ == "__main__": + print( + json.dumps( + asyncio.run(execute()), + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + ) diff --git a/tools/conformance/python_persistence_report.py b/tools/conformance/python_persistence_report.py old mode 100644 new mode 100755 index 9d443f0..b3f45c2 --- a/tools/conformance/python_persistence_report.py +++ b/tools/conformance/python_persistence_report.py @@ -110,4 +110,11 @@ async def execute() -> dict[str, Any]: if __name__ == "__main__": - print(json.dumps(asyncio.run(execute()), ensure_ascii=False, sort_keys=True, separators=(",", ":"))) + print( + json.dumps( + asyncio.run(execute()), + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + ) diff --git a/tools/conformance/python_primitives_report.py b/tools/conformance/python_primitives_report.py index bf65352..535fd23 100755 --- a/tools/conformance/python_primitives_report.py +++ b/tools/conformance/python_primitives_report.py @@ -9,7 +9,6 @@ from graph_engineering import evaluate_settled_barrier - ROOT = Path(__file__).resolve().parents[2] CASE_PATH = ROOT / "spec" / "conformance" / "settled-barrier.case.json" diff --git a/tools/conformance/python_report.py b/tools/conformance/python_report.py old mode 100644 new mode 100755 index 28fa47e..c2b8f30 --- a/tools/conformance/python_report.py +++ b/tools/conformance/python_report.py @@ -8,7 +8,6 @@ from graph_engineering import canonical_sha256, try_compile_graph - ROOT = Path(__file__).resolve().parents[2] FIXTURES = ROOT / "spec" / "conformance" @@ -19,9 +18,18 @@ def main() -> None: for name in expected["compilation"]: document = json.loads((FIXTURES / name).read_text(encoding="utf-8")) result = try_compile_graph(document) + # TypeScript only captures/canonicalizes a document after schema validation. + # GE1007 therefore has no canonical artifact, while topology diagnostics + # (cycles, reachability, missing endpoints, and so on) retain the hash of + # the already validated graph snapshot. + canonical_sha = ( + None + if any(item.code.value == "GE1007_INVALID_GRAPH" for item in result.diagnostics) + else canonical_sha256(document) + ) report[name] = { "valid": result.valid, - "canonicalSha256": canonical_sha256(document), + "canonicalSha256": canonical_sha, "diagnosticCodes": [item.code.value for item in result.diagnostics], "topologicalLayers": ( [list(layer) for layer in result.graph.topological_layers] diff --git a/tools/conformance/python_router_report.py b/tools/conformance/python_router_report.py index a8cfda9..0a672be 100755 --- a/tools/conformance/python_router_report.py +++ b/tools/conformance/python_router_report.py @@ -9,7 +9,6 @@ from graph_engineering import evaluate_route_selection - ROOT = Path(__file__).resolve().parents[2] CASE_PATH = ROOT / "spec" / "conformance" / "route-selection.case.json" diff --git a/tools/conformance/python_runtime_extended_report.py b/tools/conformance/python_runtime_extended_report.py old mode 100644 new mode 100755 index a126570..5097e85 --- a/tools/conformance/python_runtime_extended_report.py +++ b/tools/conformance/python_runtime_extended_report.py @@ -95,4 +95,11 @@ async def execute() -> dict[str, Any]: if __name__ == "__main__": - print(json.dumps(asyncio.run(execute()), ensure_ascii=False, sort_keys=True, separators=(",", ":"))) + print( + json.dumps( + asyncio.run(execute()), + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + ) diff --git a/tools/conformance/python_runtime_report.py b/tools/conformance/python_runtime_report.py old mode 100644 new mode 100755 index e945b68..c013cb8 --- a/tools/conformance/python_runtime_report.py +++ b/tools/conformance/python_runtime_report.py @@ -10,7 +10,6 @@ from graph_engineering import NodeContext, compile_graph, run_graph - ROOT = Path(__file__).resolve().parents[2] CASE_PATH = ROOT / "spec" / "conformance" / "runtime-ready-queue.case.json" @@ -50,4 +49,11 @@ async def handler( if __name__ == "__main__": - print(json.dumps(asyncio.run(execute()), ensure_ascii=False, sort_keys=True, separators=(",", ":"))) + print( + json.dumps( + asyncio.run(execute()), + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + ) diff --git a/tools/conformance/run.mjs b/tools/conformance/run.mjs index b0eae77..701c50d 100644 --- a/tools/conformance/run.mjs +++ b/tools/conformance/run.mjs @@ -86,9 +86,15 @@ if (pythonRuntime.status !== 0) { throw new Error(`Python runtime conformance failed:\n${pythonRuntime.stderr || pythonRuntime.stdout}`); } -const { runGraph } = await import( +const runtime = await import( pathToFileURL(join(root, "packages", "runtime", "dist", "index.js")).href ); +const { + decodeDurableJson, + resumeDurableGraphRun, + runGraph, + startDurableGraphRun, +} = runtime; const tsCompletion = []; const tsInputs = {}; const nodeExecutors = {}; @@ -409,3 +415,291 @@ assert.equal(tsPersistence.checkpoint.restartLoadMatches, true); assert.equal(tsPersistence.unsafeIdentifierCode, "UNSAFE_IDENTIFIER"); process.stdout.write("Cross-language event/checkpoint persistence conformance passed.\n"); + +const durableCase = JSON.parse( + await readFile(join(fixtureRoot, "durable-resume.case.json"), "utf8"), +); +const pythonDurable = spawnSync( + "uv", + ["run", "--project", "python", "python", "tools/conformance/python_durable_report.py"], + { cwd: root, encoding: "utf8" }, +); +if (pythonDurable.status !== 0) { + throw new Error( + `Python durable-recovery conformance failed:\n${pythonDurable.stderr || pythonDurable.stdout}`, + ); +} +const pyDurableWithHistory = JSON.parse(pythonDurable.stdout); +const { preCrashEvents, ...pyDurable } = pyDurableWithHistory; +const durableStore = new persistence.MemoryEventStore(); +await durableStore.append(durableCase.runId, -1, preCrashEvents); +const beforeDurableResume = await collectEvents(durableStore.read(durableCase.runId)); +const durableCalls = []; +const durableActivityKeys = []; +let durableMergeInput; +const durableResult = await resumeDurableGraphRun(durableCase.graph, { + runId: durableCase.runId, + implementationId: durableCase.implementationId, + eventStore: durableStore, + now: () => new Date("2026-07-26T12:00:00.000Z"), + nodeExecutors: { + left: () => { + throw new Error("resume reran the committed left node"); + }, + right: ({ node, attempt, activityKey }) => { + durableCalls.push({ nodeId: node.id, attempt }); + durableActivityKeys.push(activityKey); + return durableCase.resumeReturns.right; + }, + merge: ({ node, attempt, input }) => { + durableCalls.push({ nodeId: node.id, attempt }); + durableMergeInput = input; + return durableCase.resumeReturns.merge; + }, + }, +}); +const afterDurableResume = await collectEvents(durableStore.read(durableCase.runId)); +const runResumed = afterDurableResume.find(({ type }) => type === "RunResumed"); +assert.ok(runResumed, "durable history must contain RunResumed"); +const rightSchedules = afterDurableResume.filter( + ({ type, nodeId }) => type === "NodeScheduled" && nodeId === "right", +); +let terminalExecutorCalls = 0; +const terminalVersion = afterDurableResume.length; +const terminalResult = await resumeDurableGraphRun(durableCase.graph, { + runId: durableCase.runId, + implementationId: durableCase.implementationId, + eventStore: durableStore, + now: () => new Date("2026-07-26T12:00:00.000Z"), + executors: Object.fromEntries( + durableCase.graph.nodes.map(({ kind }) => [kind, () => { + terminalExecutorCalls += 1; + throw new Error("terminal resume invoked an executor"); + }]), + ), +}); +const finalDurableHistory = await collectEvents(durableStore.read(durableCase.runId)); +const tsDurable = { + status: durableResult.status, + output: durableResult.output ?? {}, + nodeStatuses: Object.fromEntries( + durableResult.nodes.map(({ nodeId, status }) => [nodeId, status]), + ), + attempts: Object.fromEntries( + durableResult.nodes.map(({ nodeId, attempts }) => [nodeId, attempts]), + ), + totalAttempts: durableResult.totalAttempts, + executorCalls: durableCalls, + mergeInput: durableMergeInput, + reusedNodeIds: runResumed.data.reusedNodeIds, + interruptedNodeIds: runResumed.data.interruptedNodeIds, + preCrashEventTypes: beforeDurableResume.map(({ type }) => type), + resumeEventTypes: afterDurableResume.slice(beforeDurableResume.length).map(({ type }) => type), + activityKeyStable: rightSchedules.length === 2 && + rightSchedules[0].data.activityKey === rightSchedules[1].data.activityKey && + isDeepStrictEqual(durableActivityKeys, [rightSchedules[0].data.activityKey]), + terminalResume: { + newEvents: finalDurableHistory.length - terminalVersion, + executorCalls: terminalExecutorCalls, + sameResult: isDeepStrictEqual(terminalResult, durableResult), + }, +}; + +assert.deepEqual(tsDurable, pyDurable, "TypeScript/Python durable recovery reports differ"); +assert.equal(tsDurable.status, durableCase.expect.status); +assert.deepEqual(tsDurable.output, durableCase.expect.output); +assert.deepEqual(tsDurable.executorCalls, durableCase.expect.executorCalls); +assert.deepEqual(tsDurable.mergeInput, durableCase.expect.mergeInput); +assert.deepEqual(tsDurable.reusedNodeIds, durableCase.expect.reusedNodeIds); +assert.deepEqual(tsDurable.interruptedNodeIds, durableCase.expect.interruptedNodeIds); +assert.equal(tsDurable.totalAttempts, durableCase.expect.totalAttempts); +assert.deepEqual(tsDurable.terminalResume, { + ...durableCase.expect.terminalResume, + sameResult: true, +}); +assert.equal(tsDurable.activityKeyStable, true); + +process.stdout.write("Cross-language event-sourced durable recovery conformance passed.\n"); + +function normalizeDurableInteropResult(result) { + return { + status: result.status, + nodeStatuses: Object.fromEntries(result.nodes.map(({ nodeId, status }) => [nodeId, status])), + failureCodes: result.failures.map(({ code }) => code), + attempts: Object.fromEntries(result.nodes.map(({ nodeId, attempts }) => [nodeId, attempts])), + outputs: result.output ?? {}, + totalAttempts: result.totalAttempts, + }; +} + +function durableSettledSemantics(events) { + const settled = events.find(({ type }) => type === "NodeSettledWithoutAttempt"); + if (settled === undefined) return null; + const result = decodeDurableJson(settled.data.result); + assert.ok(result !== null && typeof result === "object" && !Array.isArray(result)); + assert.ok( + result.failure !== null && typeof result.failure === "object" && !Array.isArray(result.failure), + ); + return { + nodeId: result.nodeId, + status: result.status, + attempts: result.attempts, + hasInput: Object.hasOwn(result, "input"), + input: result.input, + failureCode: result.failure.code, + }; +} + +const durableInteropNode = { + id: "root", + kind: "agent", + inputSchema: {}, + outputSchema: {}, + config: {}, + sideEffects: "none", +}; +const durableInteropBaseGraph = { + apiVersion: "graphengineering.reacher-z.github.io/v1alpha1", + kind: "Graph", + metadata: { name: "durable-terminal-interop", version: "1" }, + inputSchema: {}, + outputSchema: {}, + entrypoints: ["root"], + outputs: { result: { node: "root" } }, + nodes: [durableInteropNode], + edges: [], +}; +const durableInteropCases = [ + { + name: "missingExecutor", + mode: "missingExecutor", + graph: durableInteropBaseGraph, + graphInput: { realNull: null }, + implementationId: "interop@1", + tsRunId: "ts-to-python-missing-executor", + pythonRunId: "python-to-ts-missing-executor", + fixedTime: "2026-07-26T12:00:00.000Z", + expectedEventTypes: [ + "RunCreated", "RunStarted", "NodeSettledWithoutAttempt", "RunFailed", + ], + expectedSettled: { + nodeId: "root", + status: "failed", + attempts: 0, + hasInput: true, + input: { realNull: null }, + failureCode: "EXECUTOR_NOT_FOUND", + }, + }, + { + name: "outputBinding", + mode: "outputBinding", + graph: { + ...durableInteropBaseGraph, + metadata: { name: "durable-output-binding-interop", version: "1" }, + outputs: { result: { node: "root", port: "missing" } }, + }, + graphInput: { seed: 1 }, + implementationId: "interop@1", + tsRunId: "ts-to-python-output-binding", + pythonRunId: "python-to-ts-output-binding", + fixedTime: "2026-07-26T12:00:00.000Z", + expectedEventTypes: [ + "RunCreated", "RunStarted", "NodeScheduled", "NodeStarted", "NodeSucceeded", "RunFailed", + ], + expectedSettled: null, + }, +]; + +const tsProducedInterop = {}; +for (const testCase of durableInteropCases) { + const store = new persistence.MemoryEventStore(); + const options = { + runId: testCase.tsRunId, + implementationId: testCase.implementationId, + eventStore: store, + now: () => new Date(testCase.fixedTime), + ...(testCase.mode === "outputBinding" + ? { nodeExecutors: { root: () => ({ actual: true }) } } + : {}), + }; + const result = await startDurableGraphRun(testCase.graph, testCase.graphInput, options); + const events = await collectEvents(store.read(testCase.tsRunId)); + tsProducedInterop[testCase.name] = { + result: normalizeDurableInteropResult(result), + eventTypes: events.map(({ type }) => type), + settled: durableSettledSemantics(events), + events, + }; +} + +const pythonDurableInterop = spawnSync( + "uv", + ["run", "--project", "python", "python", "tools/conformance/python_durable_interop.py"], + { + cwd: root, + encoding: "utf8", + input: JSON.stringify({ + cases: durableInteropCases.map((testCase) => ({ + ...testCase, + tsEvents: tsProducedInterop[testCase.name].events, + })), + }), + }, +); +if (pythonDurableInterop.status !== 0) { + throw new Error( + `Python durable terminal interop failed:\n${pythonDurableInterop.stderr || pythonDurableInterop.stdout}`, + ); +} +const pyDurableInterop = JSON.parse(pythonDurableInterop.stdout); + +for (const testCase of durableInteropCases) { + const tsProduced = tsProducedInterop[testCase.name]; + const pythonReport = pyDurableInterop[testCase.name]; + assert.deepEqual(tsProduced.eventTypes, testCase.expectedEventTypes); + assert.deepEqual(tsProduced.settled, testCase.expectedSettled); + assert.deepEqual( + pythonReport.pythonConsumedTs.result, + tsProduced.result, + `${testCase.name}: Python did not reconstruct the TypeScript terminal result`, + ); + assert.deepEqual(pythonReport.pythonConsumedTs.eventTypes, tsProduced.eventTypes); + assert.deepEqual(pythonReport.pythonConsumedTs.settled, tsProduced.settled); + assert.deepEqual(pythonReport.pythonConsumedTs.terminalResume, { + newEvents: 0, + executorCalls: 0, + }); + + const pythonProduced = pythonReport.pythonProduced; + assert.deepEqual(pythonProduced.eventTypes, testCase.expectedEventTypes); + assert.deepEqual(pythonProduced.settled, testCase.expectedSettled); + const store = new persistence.MemoryEventStore(); + await store.append(testCase.pythonRunId, -1, pythonProduced.events); + const before = pythonProduced.events.length; + let executorCalls = 0; + const resumed = await resumeDurableGraphRun(testCase.graph, { + runId: testCase.pythonRunId, + implementationId: testCase.implementationId, + eventStore: store, + now: () => new Date(testCase.fixedTime), + nodeExecutors: { + root: () => { + executorCalls += 1; + throw new Error("terminal cross-language resume invoked an executor"); + }, + }, + }); + const after = await collectEvents(store.read(testCase.pythonRunId)); + assert.deepEqual( + normalizeDurableInteropResult(resumed), + pythonProduced.result, + `${testCase.name}: TypeScript did not reconstruct the Python terminal result`, + ); + assert.equal(after.length - before, 0); + assert.equal(executorCalls, 0); +} + +process.stdout.write( + `Cross-language terminal durable-history interop passed for ${durableInteropCases.length} cases in both directions.\n`, +); From d4de336eade9e468a219a595c45b6db8d20d74de Mon Sep 17 00:00:00 2001 From: reacher-z Date: Sun, 26 Jul 2026 07:55:36 -0700 Subject: [PATCH 2/2] docs: record durable recovery delivery --- codex_logs/daily/2026-07-26.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/codex_logs/daily/2026-07-26.md b/codex_logs/daily/2026-07-26.md index fbda595..5e54b79 100644 --- a/codex_logs/daily/2026-07-26.md +++ b/codex_logs/daily/2026-07-26.md @@ -236,3 +236,18 @@ Canonical IR -> language models/builders -> compiler -> deterministic scheduler - Distributed leases/fencing, checkpoint acceleration, replay/fork, approval callbacks, and universal exactly-once effects remain deliberately unclaimed follow-up work. Registry publication remains deferred. + +## 14:55 UTC durable-recovery delivery checkpoint + +- Committed the integrated feature as + `854b2e3a35e36c2f41a82704485934a96cce644c` with author and committer + `reacher-z ` and no co-author trailer, then pushed + `feat/durable-recovery` to the existing GitHub remote. +- Opened draft pull request + [#14](https://github.com/reacher-z/GraphEngineering/pull/14) against protected + `main`. The GitHub App lacked collaborator authority for PR creation, so the + authenticated `gh` fallback completed the same draft-only operation. +- Every reported remote CI, package, cross-language, progress-scanner, + dependency-review, and Python/JavaScript CodeQL check passed. The pull request + remains intentionally unmerged and review-required; protected-branch policy + was not bypassed.