diff --git a/.github/workflows/deploy-services.yml b/.github/workflows/deploy-services.yml index dbc20ee6..9a98c541 100644 --- a/.github/workflows/deploy-services.yml +++ b/.github/workflows/deploy-services.yml @@ -146,18 +146,11 @@ jobs: with: ref: ${{ github.event.workflow_run.head_sha || github.sha }} - - uses: pnpm/action-setup@v6 - + # No pnpm install / workspace build here: the graph canary moved to its + # own job below, and check-log-smoke.mjs imports only node:crypto. - uses: actions/setup-node@v7 with: node-version: '24' - cache: pnpm - - - name: Install canary dependencies - run: pnpm install --frozen-lockfile - - - name: Build canary dependencies - run: pnpm --filter @atrib/mcp build - uses: superfly/flyctl-actions/setup-flyctl@ed8efb33836e8b2096c7fd3ba1c8afe303ebbff1 @@ -170,9 +163,6 @@ jobs: shell: bash run: node .github/scripts/check-log-smoke.mjs - - name: Smoke test log to graph indexing - run: pnpm --filter @atrib/log-node graph-canary - deploy-graph-node: needs: changes if: needs.changes.outputs.graph_node == 'true' @@ -189,6 +179,60 @@ jobs: FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} run: flyctl deploy -c services/graph-node/fly.toml --remote-only + # The log-to-graph indexing canary runs in its own job so it starts AFTER + # both deploys settle rather than alongside them. + # + # It used to be the last step of deploy-log-node, which depends only on + # `changes`, so deploy-log-node and deploy-graph-node ran in parallel with no + # ordering. A commit touching a shared path (packages/mcp/*, package.json, + # pnpm-lock.yaml, pnpm-workspace.yaml, tsconfig.base.json, .dockerignore) + # sets both flags, and the canary could then poll graph-node while its + # machine was restarting into the new release. graph-node replays its whole + # record archive on start, so that window serves 502s and the canary fails on + # a system that is actually fine. + # + # It now also runs when only graph-node deployed. The canary exercises log + # submit through graph index, so a graph-node-only release is exactly the + # change it should gate; previously it ran on log-node releases only, which + # is why the 2026-07-29 trace regression shipped without the canary seeing it. + graph-canary: + needs: [changes, deploy-log-node, deploy-graph-node] + # !cancelled() keeps a skipped sibling deploy from skipping this job, since + # a skipped `needs` entry otherwise cascades, while still standing down when + # the run itself is cancelled. The guards then re-impose the real + # conditions: `changes` actually ran, at least one of the two services + # deployed, and neither deploy failed or was cancelled (a red canary after a + # failed deploy would only restate the deploy failure). + if: >- + !cancelled() + && needs.changes.result == 'success' + && (needs.changes.outputs.log_node == 'true' || needs.changes.outputs.graph_node == 'true') + && needs.deploy-log-node.result != 'failure' + && needs.deploy-graph-node.result != 'failure' + && needs.deploy-log-node.result != 'cancelled' + && needs.deploy-graph-node.result != 'cancelled' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ github.event.workflow_run.head_sha || github.sha }} + + - uses: pnpm/action-setup@v6 + + - uses: actions/setup-node@v7 + with: + node-version: '24' + cache: pnpm + + - name: Install canary dependencies + run: pnpm install --frozen-lockfile + + - name: Build canary dependencies + run: pnpm --filter @atrib/mcp build + + - name: Smoke test log to graph indexing + run: pnpm --filter @atrib/log-node graph-canary + deploy-directory-node: needs: changes if: needs.changes.outputs.directory_node == 'true' diff --git a/CLAUDE.md b/CLAUDE.md index 9f195455..620375bf 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -52,7 +52,7 @@ atrib/ CLAUDE.md # THIS FILE: hub doc, conventions, invariants DESIGN.md # Product design system source of truth: current state, target state, tokens, components, surface backlog atrib-spec.md # The single source of truth for the protocol - DECISIONS.md # Architectural decision log (D001-D187; D070 accepts the Record Body Archive Layer per §2.12; D073 remains the placeholder ADR for the `handoff` event_type byte; D071 codifies spec writing conventions; D072 governs orphan handling; D074 + D075 capture the git-trailer integration and compose-not-override hook config patterns; D076 introduces the opt-in long-lived atrib-emit daemon; D077 codifies pass^k as the primary Track B reporting metric, k=3 default, promoted from P019 on 2026-05-10; D081 introduces emitInProcess for hook-class producers (signing in-process, byte-identical to MCP-signed records); D082 supersedes D081's integration shape by shipping `atrib-emit-cli` from `@atrib/emit` and having the hook helper spawn the binary, so the hook source directory never becomes an npm workspace; D083 extends D078 with a harness session-id discovery registry in `@atrib/mcp` so cognitive-primitive MCP servers spawned by Claude Code, and future harnesses, derive `context_id` from the parent's session_id env var without operator-side config (v2 2026-05-23 adds optional file-fallback to the registry for startup-spawn harnesses like Claude Code whose MCP children pre-date the per-session env; SessionStart hook writes `~/.claude/state/active-session-id-` and `resolveEnvContextId` reads it when env unset); D084 ships per-event instrumentation jsonl files under `~/.atrib/state/` for read primitives (Surface 6, via `logReadPrimitiveCall` in `@atrib/mcp@0.10.0`), SessionStart (Surface 7), cli-spawn transport (Surface 8), plus a unified analyzer (Surface 9, host-side) that joins all four jsonl pillars + the signed-record mirror so the loop-closure question can be answered against data instead of inferred from one source; `@atrib/recall` compact-mode response now always carries `record_hash` so callers can chain other primitives from any result; D085/D086 tune recall calibration and BM25 content indexing; D087 formalizes signed diagnostic outcome + trace replay as the canonical repair/refinement pattern; D088 updates AP2 transaction detection to successful CheckoutReceipt / PaymentReceipt as the current v0.2 hook while keeping the v0.1 PaymentMandate DataPart fallback; D089 adds verifier-side AP2 / Verifiable Intent evidence checks in `@atrib/verify`; D090 adds async AP2 receipt JWT verification with `jose`, trusted JWKS, and verifier metadata; D091 adds async AP2 / VI SD-JWT conformance with OpenWallet `sd-jwt-js`; D092 adds typed AP2 / VI mandate constraint evaluation; D093 makes the AP2 / VI fixture directory the local verifier corpus; D094 attaches AP2 / VI evidence to verifier results as a tiered block; D095 adds an AP2 Path 2 content_id receipt identity ladder; D096 adds a pinned offline AP2 / VI crypto conformance corpus and named verifier hardening for JOSE, JWKS, SD-JWT, and clock edges; D097 adds an opt-in AP2 live interop artifact harness in `@atrib/integration`; D098 keeps AP2 receipt signatures as external evidence and makes Path 2 transaction records carry an agent `signers[]` entry via `signTransactionRecord`; D099 commits explicit emit content through default `args_hash` and gives direct CLI emits a default mirror path; D100 lets MCP middleware sign and run `onRecord` with log submission disabled for offline tests and local-mirror-only hosts; D101 adds the substrate-wide adversarial conformance corpus for §1.4 signing, §3.2.4 full edge derivation, D067 race vectors, and D052 creator-signer separation; D102 requires sandboxed producers to keep Ed25519 signing keys outside the sandbox through a host signer proxy; D103 adds optional SSE and JSON Feed subscription surfaces over commitment-visible fields; D104 codifies `ATRIB_PARENT_RECORD_HASH` parent-child threading through `informed_by`; D105 adds verifier-side Pattern 3 handoff claim acceptance in `@atrib/verify`; D106 promotes `@atrib/verify-mcp` as cognitive primitive #7 after two independent Pattern 3 receiving flows; D107 adds AP2 counterparty attestation over atrib transaction bytes and makes `cross_attestation.signers_valid` count distinct verified keys; D108 makes OpenTelemetry/OpenInference span trees an intake/correlation layer and puts detailed observability fields in local sidecar content for recall, trace, and summarize; D109 adds generic tiered authorization evidence blocks, MCP/OAuth evidence checks, and resolved capability facts to `@atrib/verify`; D110 closes producer-side MCP/OAuth evidence capture, DPoP proof checks, the local harness, and the §5.5.6 OAuth corpus; D111 adds the archive evidence API, producer-side archive submission, HTTP-backed DPoP replay-cache adapter, and host-owned OAuth introspection helper; D112 adds `@atrib/memory-tool` as an Anthropic Memory Tool handler wrapper that signs memory commands while leaving storage host-owned; D113 makes producers omit unvalidated `informed_by` refs by default while keeping `allow_unresolved_informed_by` for deliberate dangling claims; D114 adds the Google ADK Python plugin proof at the Python tool callback boundary; D115 defines the three-signal same-session agent-to-subagent env bundle; D116 adds source-aware producer-side `informed_by` validation; D117 classifies demo/proof record treatment by execution surface; D118 makes the explorer primary trace path a presentation rule over trace plus chain while keeping human-attested approval as separate signed evidence; D119 keeps AAuth as verifier-side authorization evidence with a producer capture helper and conformance corpus; D120 accepts the optional local substrate coordinator pattern while keeping startup-spawn signing, sidecars, and outbound context wrapper-owned; D121 accepts runtime-log proof manifests for host-owned run windows while keeping raw runtime logs outside the public log and trace systems as projections; D122 keeps host runtime adapters distinct from agent framework adapters and pins one signing owner per host event; D123 makes critical-path content recall require complete evidence or explicit fallback; D124 makes base recall context scope explicit; D125 makes complete content recall coverage-first; D126 adds the durable content index; D127 gates primitive runtime health on recall contract freshness; D128 makes host-owned primitive runtime updates a build, restart, and direct-probe command; D129 gates primitive runtime health on every mounted primitive package and tool surface; D130 gates primitive runtime health on deterministic non-mutating behavioral probes while write primitives stay skipped until validate-only contracts exist; D131 adds the Google ADK decision-ledger proof at the before-tool authority boundary; D132 keeps x401 proof requirements as verifier-side authorization evidence rather than payment detection; D133 adds `@atrib/action-gate` as the host-owned decision and outcome proof package for high-impact actions; D134 adds x401 producer capture, local proof-gate and multi-endpoint E2E, sanitized archive projection, Explorer propagation, optional origin, trust, proof-payment binding facts, released Proof SDK native interop, Proof VC credential-verifier evidence, and the Proof upstream open-thread map; D135 threads delegated-builder atrib context via orchestrator-injected explicit `context_id` + `informed_by` rather than ambient env/file, since the builder's atrib runtime is a shared persistent process; D136 adds the consolidated client SDKs: `@atrib/sdk` with attest/recall verbs, daemon-first with in-process fallback and no third signing implementation, plus the Python `atrib` distribution as the first non-TypeScript §1/§5 implementation, held byte-identical through the shared conformance corpora and a cross-implementation determinism harness, both in-repo with an extraction-ready layout; D137 makes the universal evidence envelope the single attachment model for external evidence per §5.5.7; D138 generalizes cross-log replication to anchor plurality per §2.11.7-§2.11.13; D139 adds the session_checkpoint event type extension-first per §1.2.10 with byte 0x08 reserved; D140 adds delegation certificates, principal keys certifying ephemeral run keys, per §1.11; D141 publishes the dev.atrib/attribution MCP extension v0.1 per §1.5.4.1 and docs/extensions/dev.atrib-attribution/; D142 codifies orchestration-topology baton-pass and join records as conventional attest content shapes and registers the continuation-packet evidence profile, the ninth atrib-maintained §5.5.7 profile; D143 evaluates authority as verifier-side policy over informed_by lineage with minimum-along-path propagation, closing the laundering gap without adding record fields or edge types; D144 adds the pre-action elevation gate in @atrib/action-gate composing D143 authority evaluation with the D052/D107 cross-attestation distinctness rule; D145 adds action-bound single-use authorization tokens over a JCS-plus-SHA-256 five-field binding with a host-owned atomic consumption store; D146 widens mirror inheritance from the effective file to the corpus-scoped directory, deterministic cross-file tie-breaks by signed timestamp then canonical hash, with a deletable advisory tail index; D147 spins the payment-rail material out of the spec core into the independently versioned payments profile at docs/payments-profile.md, leaving stable tombstone anchors behind and registering the payments-detection and payments-settlement evidence profiles; D148 promotes the primitive runtime to atribd, the public stateless-native local daemon, built behind a transport adapter with per-context write serialization and the seven-family conformance corpus at spec/conformance/atribd/; D149 composes cross-attestation with a caller-supplied trust set so untrusted co-signers surface as sybil_suspected instead of counting as authority; D150 generalizes attestation as corroboration off transactions to any signed target, extension-first per §8.7.6; D151 makes own signed content win over chain-derived text in rendering; D152 makes handoff verdicts receiver-computed; D153 gives chain expansion a reserved budget share; D154 separates memory retrieval selection from expansion; D155 makes write-primitive refusals error-shaped; D156 makes tool-call score suppression liftable per query; D157 renders revision lineage as an ordered connected chain; D158 renders dropped chain members as a compact ordered path line; D159 lets the lineage tail win budget admission; D160 carries temporal provenance on rendered memory lines; D161 links cross-topic revisions under a stricter content bar; D162 keeps factual values untruncated in rendered memory; D163 ships the session-transcript runtime-log source that binds harness transcript windows (Claude-Code-style session JSONL, the Session Traces Format shape; D164 collapses the primitive surface to the attest/recall verbs with the fifteen legacy tool names as permanent aliases, byte-identical records, and the alias-window migration rules) to signed records through manifest receipts, fork and compaction bindings, and durable-body refs; v2 adds the auto-detected codex-rollout-jsonl profile; D165 wires D142 baton/join signing into the operator.s orchestration routing plane with per-leg routing facts in content/sidecar, adds the `subagent` role term, and gives §6.7.1 envelopes / §1.11 certificate scopes an optional cost_policy (model_tiers, max_tokens) as a signal-only signed budget grant; D166 makes verifiable-action the @atrib/mcp-wrap default, with signed tool, argument, and result commitments plus linked request/outcome records for pre-call receipt injection). A "Pending decisions" section at the end (forward-looking pattern) tracks forward-looking decisions awaiting action (P002, P004, P005, P008, P009, P010, P012, P013, P016-P018, P021, P024, P026, P027, P036-P040, P047, P052, P053, P054-P058). P012, P013, P016, and P017 cover remaining runtime and sandboxing patterns. P018 and P021 cover eval-framework adoption and benchmark publication. P024, P026, and P027 cover spec hosting, multi-creator SessionStart, and host-side hook deployment. P036-P040 cover the support/RCA implication set from the Autumn support-investigation case study: cross-harness continuation packets, skill/context provenance, hosted-agent diagnostics, the support/RCA demo wedge, and Mastra source verification. P047 remains from the 2026-07 redesign upgrade-path candidate set (attest/recall rename; full drafts under `docs/adr-draft-p04x-*.md`, plan in `docs/redesign-upgrade-path.md`); P042-P045 and P049 were promoted to D137-D141 on 2026-07-06; P050 was promoted to D142 on 2026-07-08; P048 was promoted to D147 and P046 to D148 on 2026-07-10. P051 was promoted to D165 on 2026-07-15. P053 holds multi-head revision merge semantics behind an independent-application gate. P054-P058 port the benchmark-derived substrate findings first filed on 2026-07-17, renumbered from P043-P047 because those numbers were promoted in the interim: the hidden-superseded prior_position projection leak, question-form recall calibration, attest indexability warnings, supersession trigger policy beyond agent errors, and the local-mirror-only SDK mode ADR. P052 holds the transcript recall corpus composition-first: manifests and extraction receipts are the interface to memory tools, with explicit attribution triggers for ever revisiting. + DECISIONS.md # Architectural decision log (D001-D187; D070 accepts the Record Body Archive Layer per §2.12; D073 remains the placeholder ADR for the `handoff` event_type byte; D071 codifies spec writing conventions; D072 governs orphan handling; D074 + D075 capture the git-trailer integration and compose-not-override hook config patterns; D076 introduces the opt-in long-lived atrib-emit daemon; D077 codifies pass^k as the primary Track B reporting metric, k=3 default, promoted from P019 on 2026-05-10; D081 introduces emitInProcess for hook-class producers (signing in-process, byte-identical to MCP-signed records); D082 supersedes D081's integration shape by shipping `atrib-emit-cli` from `@atrib/emit` and having the hook helper spawn the binary, so the hook source directory never becomes an npm workspace; D083 extends D078 with a harness session-id discovery registry in `@atrib/mcp` so cognitive-primitive MCP servers spawned by Claude Code, and future harnesses, derive `context_id` from the parent's session_id env var without operator-side config (v2 2026-05-23 adds optional file-fallback to the registry for startup-spawn harnesses like Claude Code whose MCP children pre-date the per-session env; SessionStart hook writes `~/.claude/state/active-session-id-` and `resolveEnvContextId` reads it when env unset); D084 ships per-event instrumentation jsonl files under `~/.atrib/state/` for read primitives (Surface 6, via `logReadPrimitiveCall` in `@atrib/mcp@0.10.0`), SessionStart (Surface 7), cli-spawn transport (Surface 8), plus a unified analyzer (Surface 9, host-side) that joins all four jsonl pillars + the signed-record mirror so the loop-closure question can be answered against data instead of inferred from one source; `@atrib/recall` compact-mode response now always carries `record_hash` so callers can chain other primitives from any result; D085/D086 tune recall calibration and BM25 content indexing; D087 formalizes signed diagnostic outcome + trace replay as the canonical repair/refinement pattern; D088 updates AP2 transaction detection to successful CheckoutReceipt / PaymentReceipt as the current v0.2 hook while keeping the v0.1 PaymentMandate DataPart fallback; D089 adds verifier-side AP2 / Verifiable Intent evidence checks in `@atrib/verify`; D090 adds async AP2 receipt JWT verification with `jose`, trusted JWKS, and verifier metadata; D091 adds async AP2 / VI SD-JWT conformance with OpenWallet `sd-jwt-js`; D092 adds typed AP2 / VI mandate constraint evaluation; D093 makes the AP2 / VI fixture directory the local verifier corpus; D094 attaches AP2 / VI evidence to verifier results as a tiered block; D095 adds an AP2 Path 2 content_id receipt identity ladder; D096 adds a pinned offline AP2 / VI crypto conformance corpus and named verifier hardening for JOSE, JWKS, SD-JWT, and clock edges; D097 adds an opt-in AP2 live interop artifact harness in `@atrib/integration`; D098 keeps AP2 receipt signatures as external evidence and makes Path 2 transaction records carry an agent `signers[]` entry via `signTransactionRecord`; D099 commits explicit emit content through default `args_hash` and gives direct CLI emits a default mirror path; D100 lets MCP middleware sign and run `onRecord` with log submission disabled for offline tests and local-mirror-only hosts; D101 adds the substrate-wide adversarial conformance corpus for §1.4 signing, §3.2.4 full edge derivation, D067 race vectors, and D052 creator-signer separation; D102 requires sandboxed producers to keep Ed25519 signing keys outside the sandbox through a host signer proxy; D103 adds optional SSE and JSON Feed subscription surfaces over commitment-visible fields; D104 codifies `ATRIB_PARENT_RECORD_HASH` parent-child threading through `informed_by`; D105 adds verifier-side Pattern 3 handoff claim acceptance in `@atrib/verify`; D106 promotes `@atrib/verify-mcp` as cognitive primitive #7 after two independent Pattern 3 receiving flows; D107 adds AP2 counterparty attestation over atrib transaction bytes and makes `cross_attestation.signers_valid` count distinct verified keys; D108 makes OpenTelemetry/OpenInference span trees an intake/correlation layer and puts detailed observability fields in local sidecar content for recall, trace, and summarize; D109 adds generic tiered authorization evidence blocks, MCP/OAuth evidence checks, and resolved capability facts to `@atrib/verify`; D110 closes producer-side MCP/OAuth evidence capture, DPoP proof checks, the local harness, and the §5.5.6 OAuth corpus; D111 adds the archive evidence API, producer-side archive submission, HTTP-backed DPoP replay-cache adapter, and host-owned OAuth introspection helper; D112 adds `@atrib/memory-tool` as an Anthropic Memory Tool handler wrapper that signs memory commands while leaving storage host-owned; D113 makes producers omit unvalidated `informed_by` refs by default while keeping `allow_unresolved_informed_by` for deliberate dangling claims; D114 adds the Google ADK Python plugin proof at the Python tool callback boundary; D115 defines the three-signal same-session agent-to-subagent env bundle; D116 adds source-aware producer-side `informed_by` validation; D117 classifies demo/proof record treatment by execution surface; D118 makes the explorer primary trace path a presentation rule over trace plus chain while keeping human-attested approval as separate signed evidence; D119 keeps AAuth as verifier-side authorization evidence with a producer capture helper and conformance corpus; D120 accepts the optional local substrate coordinator pattern while keeping startup-spawn signing, sidecars, and outbound context wrapper-owned; D121 accepts runtime-log proof manifests for host-owned run windows while keeping raw runtime logs outside the public log and trace systems as projections; D122 keeps host runtime adapters distinct from agent framework adapters and pins one signing owner per host event; D123 makes critical-path content recall require complete evidence or explicit fallback; D124 makes base recall context scope explicit; D125 makes complete content recall coverage-first; D126 adds the durable content index; D127 gates primitive runtime health on recall contract freshness; D128 makes host-owned primitive runtime updates a build, restart, and direct-probe command; D129 gates primitive runtime health on every mounted primitive package and tool surface; D130 gates primitive runtime health on deterministic non-mutating behavioral probes while write primitives stay skipped until validate-only contracts exist; D131 adds the Google ADK decision-ledger proof at the before-tool authority boundary; D132 keeps x401 proof requirements as verifier-side authorization evidence rather than payment detection; D133 adds `@atrib/action-gate` as the host-owned decision and outcome proof package for high-impact actions; D134 adds x401 producer capture, local proof-gate and multi-endpoint E2E, sanitized archive projection, Explorer propagation, optional origin, trust, proof-payment binding facts, released Proof SDK native interop, Proof VC credential-verifier evidence, and the Proof upstream open-thread map; D135 threads delegated-builder atrib context via orchestrator-injected explicit `context_id` + `informed_by` rather than ambient env/file, since the builder's atrib runtime is a shared persistent process; D136 adds the consolidated client SDKs: `@atrib/sdk` with attest/recall verbs, daemon-first with in-process fallback and no third signing implementation, plus the Python `atrib` distribution as the first non-TypeScript §1/§5 implementation, held byte-identical through the shared conformance corpora and a cross-implementation determinism harness, both in-repo with an extraction-ready layout; D137 makes the universal evidence envelope the single attachment model for external evidence per §5.5.7; D138 generalizes cross-log replication to anchor plurality per §2.11.7-§2.11.13; D139 adds the session_checkpoint event type extension-first per §1.2.10 with byte 0x08 reserved; D140 adds delegation certificates, principal keys certifying ephemeral run keys, per §1.11; D141 publishes the dev.atrib/attribution MCP extension v0.1 per §1.5.4.1 and docs/extensions/dev.atrib-attribution/; D142 codifies orchestration-topology baton-pass and join records as conventional attest content shapes and registers the continuation-packet evidence profile, the ninth atrib-maintained §5.5.7 profile; D143 evaluates authority as verifier-side policy over informed_by lineage with minimum-along-path propagation, closing the laundering gap without adding record fields or edge types; D144 adds the pre-action elevation gate in @atrib/action-gate composing D143 authority evaluation with the D052/D107 cross-attestation distinctness rule; D145 adds action-bound single-use authorization tokens over a JCS-plus-SHA-256 five-field binding with a host-owned atomic consumption store; D146 widens mirror inheritance from the effective file to the corpus-scoped directory, deterministic cross-file tie-breaks by signed timestamp then canonical hash, with a deletable advisory tail index; D147 spins the payment-rail material out of the spec core into the independently versioned payments profile at docs/payments-profile.md, leaving stable tombstone anchors behind and registering the payments-detection and payments-settlement evidence profiles; D148 promotes the primitive runtime to atribd, the public stateless-native local daemon, built behind a transport adapter with per-context write serialization and the seven-family conformance corpus at spec/conformance/atribd/; D149 composes cross-attestation with a caller-supplied trust set so untrusted co-signers surface as sybil_suspected instead of counting as authority; D150 generalizes attestation as corroboration off transactions to any signed target, extension-first per §8.7.6; D151 makes own signed content win over chain-derived text in rendering; D152 makes handoff verdicts receiver-computed; D153 gives chain expansion a reserved budget share; D154 separates memory retrieval selection from expansion; D155 makes write-primitive refusals error-shaped; D156 makes tool-call score suppression liftable per query; D157 renders revision lineage as an ordered connected chain; D158 renders dropped chain members as a compact ordered path line; D159 lets the lineage tail win budget admission; D160 carries temporal provenance on rendered memory lines; D161 links cross-topic revisions under a stricter content bar; D162 keeps factual values untruncated in rendered memory; D163 ships the session-transcript runtime-log source that binds harness transcript windows (Claude-Code-style session JSONL, the Session Traces Format shape; D164 collapses the primitive surface to the attest/recall verbs with the fifteen legacy tool names as permanent aliases, byte-identical records, and the alias-window migration rules) to signed records through manifest receipts, fork and compaction bindings, and durable-body refs; v2 adds the auto-detected codex-rollout-jsonl profile; D165 wires D142 baton/join signing into the operator.s orchestration routing plane with per-leg routing facts in content/sidecar, adds the `subagent` role term, and gives §6.7.1 envelopes / §1.11 certificate scopes an optional cost_policy (model_tiers, max_tokens) as a signal-only signed budget grant; D166 makes verifiable-action the @atrib/mcp-wrap default, with signed tool, argument, and result commitments plus linked request/outcome records for pre-call receipt injection). A "Pending decisions" section at the end (forward-looking pattern) tracks forward-looking decisions awaiting action (P002, P004, P005, P008, P009, P010, P012, P013, P016-P018, P021, P024, P026, P027, P036-P040, P047, P052, P053, P054-P059). P012, P013, P016, and P017 cover remaining runtime and sandboxing patterns. P018 and P021 cover eval-framework adoption and benchmark publication. P024, P026, and P027 cover spec hosting, multi-creator SessionStart, and host-side hook deployment. P036-P040 cover the support/RCA implication set from the Autumn support-investigation case study: cross-harness continuation packets, skill/context provenance, hosted-agent diagnostics, the support/RCA demo wedge, and Mastra source verification. P047 remains from the 2026-07 redesign upgrade-path candidate set (attest/recall rename; full drafts under `docs/adr-draft-p04x-*.md`, plan in `docs/redesign-upgrade-path.md`); P042-P045 and P049 were promoted to D137-D141 on 2026-07-06; P050 was promoted to D142 on 2026-07-08; P048 was promoted to D147 and P046 to D148 on 2026-07-10. P051 was promoted to D165 on 2026-07-15. P053 holds multi-head revision merge semantics behind an independent-application gate. P054-P058 port the benchmark-derived substrate findings first filed on 2026-07-17, renumbered from P043-P047 because those numbers were promoted in the interim: the hidden-superseded prior_position projection leak, question-form recall calibration, attest indexability warnings, supersession trigger policy beyond agent errors, and the local-mirror-only SDK mode ADR. P052 holds the transcript recall corpus composition-first: manifests and extraction receipts are the interface to memory tools, with explicit attribution triggers for ever revisiting. P059 holds the successor to graph-node's in-memory store, filed after the 2026-07-29 heap-ceiling outage; the heap watchdog's warn threshold is its intended trigger. # Decision-log continuation: D167 ships external-operator-ready witness software; D168 adds runtime-bound coverage manifests; D169 adds one-time protected MCP execution permits; D170 adds fail-closed checkpoint gossip and immutable split-view incidents; D171 adds one-command named identity plus explicit scope and trusted-time verification state; D172 publishes prior-run revocation before successor activation and propagates verified revocation views into verifiers and protected executors; D173 makes log position the sole revocation-order evidence and surfaces missing order as unverifiable; D174 adds bounded policy-state and public revision-state projections that keep forks unresolved and state their different verification bases; D175 adds exact log-index subscription cursors and native EventSource reconnect semantics; D176 makes the explorer distinguish compact log commitments from retrievable signed record bodies; D177 adds the paired SDK action-evidence path; D178 ships the open operating-graph application profile and reference client; D179 registers separate Nostr and Buzz evidence profiles and repairs the verifier registry drift; D180 gives directory anchors a durable linear commit journal with exact-body reconciliation; D181 makes directory consistency follow the verifier's exact previously accepted anchor; D182 binds witness acceptance to endpoint, trust root, liveness, and operator epoch; D183 separates cursor-bound runtime observation from durable host acceptance and semantic promotion; D184 makes stateless MCP writes duplicate-safe through action-bound idempotency keys and durable result replay. # Decision-log current head: D187 keeps one atribd daemon per operator profile. ARCHITECTURE.md # Technical architecture overview: trust model, protocol layers, design decisions PRIOR-ART.md # Prior art & standards map: every spec/protocol atrib builds on, organized by layer diff --git a/DECISIONS.md b/DECISIONS.md index 82015759..10c64e5b 100644 --- a/DECISIONS.md +++ b/DECISIONS.md @@ -10311,3 +10311,49 @@ pre-allocate. **Likely outcome (not committed):** accept at PR time; this entry exists so the ADR obligation is findable. **ADR number** will be assigned when the decision is acted on. Do not pre-allocate. + +## P059: graph-node's in-memory store needs a disk-backed successor before the heap ceiling returns + +**Filed 2026-07-29** after the graph-node outage fixed in +[#599](https://github.com/creatornader/atrib/pull/599). That PR removed the +trigger and raised the ceiling. It did not change the shape that produced the +outage, so this entry exists so the shape is not forgotten between incidents. + +**Source:** graph-node holds every record and every derived index in memory. The +live heap therefore grows linearly and without bound with the log. On 2026-07-29 +that live set reached the default V8 heap ceiling (`heap_size_limit` was 493MB on +the 1024mb machine, because Node caps old-space near half a small machine's RAM +regardless of the `[[vm]]` setting). Past that point the process aborted with +"Ineffective mark-compacts near heap limit" on any further allocation, and Fly +served the dead instance as 502s. `services/graph-node/src/persistence.ts` had +already named the boundary: the in-memory shape is "sustainable until ~10^5 +records per graph-node instance; beyond that the sustainable shape is a +disk-backed graph store." The log passed 10^5 in July 2026. + +**Measured position at filing.** 112,781 records, 439MB RSS, 780MB heap ceiling +after the fix, so roughly half the ceiling consumed. Ingest was running about 4k +to 7k records/day at roughly 3.4KB of heap each, which is 15MB to 22MB/day. That +is a runway of weeks, not quarters. The heap watchdog added alongside this entry +(`src/heap-watchdog.ts`, warn at 70%, error at 85%) exists to make the runway +observable rather than inferred, and its warning is the intended trigger for +acting on this entry. + +**The decision in question:** whether the successor is the disk-backed graph +store persistence.ts anticipates, a bounded in-memory working set over the +existing archive, sharding by `context_id` across instances, or simply +continuing to raise the ceiling with machine size. The first three change the +service's operational shape; the fourth is bounded by machine cost and only ever +defers the same failure. + +**Not urgent as a correctness matter, and not open-ended either.** Whatever is +chosen must preserve two properties the current design gets for free: [§3.2.4](atrib-spec.md#324-edge-derivation-rules) +edge derivation stays deterministic over the full record set, and the [§1.9](atrib-spec.md#19-key-rotation-and-revocation) +revocation registry stays a global scan, since a key revoked in one session +retires it everywhere. A bounded working set is the option most in tension with +both. + +**Likely outcome (not committed):** raise machine memory once more as a stopgap +when the watchdog first warns, and treat that warning as the deadline for +choosing among the three structural options rather than as the fix. + +**ADR number** will be assigned when the decision is acted on. Do not pre-allocate. diff --git a/services/graph-node/fly.toml b/services/graph-node/fly.toml index 97dbee9d..e82a03a3 100644 --- a/services/graph-node/fly.toml +++ b/services/graph-node/fly.toml @@ -42,9 +42,12 @@ primary_region = 'iad' # alone is not enough: see NODE_OPTIONS above, V8 will not use the extra RAM # unless told. Both settings move together. # - # Headroom is finite and the log only grows (~4k records/day at 2026-07-29, - # ~4KB each in heap, so ~17MB/day). At a 768MB heap that is a few weeks - # before this becomes tight again. The durable fix is the disk-backed graph - # store already flagged in src/persistence.ts, which notes this in-memory - # shape is sustainable to ~10^5 records; the log passed that in July 2026. + # Headroom is finite and the log only grows (~4k to 7k records/day at + # 2026-07-29, ~3.4KB each in heap, so ~15-22MB/day). At a 768MB heap that is + # weeks, not quarters. The durable fix is the disk-backed graph store already + # flagged in src/persistence.ts, which notes this in-memory shape is + # sustainable to ~10^5 records; the log passed that in July 2026. Choosing + # the successor is tracked as P059 in DECISIONS.md, and the heap watchdog + # (src/heap-watchdog.ts) warns at 70% so the runway is observed, not guessed. + # If you raise this, raise NODE_OPTIONS above with it or nothing changes. memory = '1024mb' diff --git a/services/graph-node/src/heap-watchdog.ts b/services/graph-node/src/heap-watchdog.ts new file mode 100644 index 00000000..8cc60aac --- /dev/null +++ b/services/graph-node/src/heap-watchdog.ts @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: Apache-2.0 + +/** + * Threshold logic for the heap-utilization watchdog in main.ts. + * + * Lives in its own module so the thresholds are unit-testable. The watchdog's + * whole value is firing at the right moment; an alerting path that quietly + * never fires is worse than no alert, because it reads as an all-clear. + * + * Context: graph-node holds every record and derived index in memory, so the + * live heap grows linearly and without bound with the log. On 2026-07-29 that + * live set reached the V8 ceiling at ~112k records and the process began + * aborting on any further allocation, which Fly served as 502s. Nothing warned + * first. + * + * Thresholds sit below the disk watchdog's 80/95 because heap has no graceful + * degradation: crossing the V8 limit aborts the process immediately, + * per-request allocation sits on top of the live set so the usable ceiling is + * under 100%, and recovery costs a full archive replay that serves errors + * throughout. + */ + +export const HEAP_WARN_FRACTION = 0.7 +export const HEAP_ERROR_FRACTION = 0.85 + +export type HeapAlertLevel = 'ok' | 'warn' | 'error' + +/** + * Classify a heap-utilization fraction (used / limit). + * + * Non-finite input (a zero limit, or NaN from a runtime that does not report + * heap statistics) classifies as 'ok': the watchdog is advisory and must never + * turn a missing measurement into a false alarm. + */ +export function heapAlertLevel(fraction: number): HeapAlertLevel { + if (!Number.isFinite(fraction)) return 'ok' + if (fraction >= HEAP_ERROR_FRACTION) return 'error' + if (fraction >= HEAP_WARN_FRACTION) return 'warn' + return 'ok' +} + +/** + * Whether a transition from `previous` to `current` should be logged. + * + * Only transitions are logged: a steady warn state would otherwise repeat every + * interval and train operators to filter the message out. Recovery to 'ok' is + * logged so the all-clear is visible, but only when something was wrong before. + */ +export function shouldLogHeapTransition( + previous: HeapAlertLevel, + current: HeapAlertLevel, +): boolean { + return current !== previous +} diff --git a/services/graph-node/src/main.ts b/services/graph-node/src/main.ts index cd5f31a6..0a88ee73 100644 --- a/services/graph-node/src/main.ts +++ b/services/graph-node/src/main.ts @@ -27,6 +27,14 @@ import { createRecordStore } from './store.js' import { createArchiveAppender, replayArchive } from './persistence.js' import { statfs } from 'node:fs/promises' import { dirname } from 'node:path' +import { getHeapStatistics } from 'node:v8' +import { + HEAP_ERROR_FRACTION, + HEAP_WARN_FRACTION, + heapAlertLevel, + shouldLogHeapTransition, + type HeapAlertLevel, +} from './heap-watchdog.js' import type { AtribRecord } from '@atrib/mcp' const port = parseInt(process.env.PORT ?? '3200', 10) @@ -124,6 +132,84 @@ if (archivePath) { handle.unref() } +// Periodic heap-utilization watchdog. Mirrors the disk watchdog above, for the +// resource that actually takes this service down. +// +// graph-node keeps every record and every derived index in memory, so the live +// heap grows linearly and without bound as the log grows. On 2026-07-29 that +// live set reached the V8 heap ceiling at ~112k records and the process began +// aborting on any further allocation ("Ineffective mark-compacts near heap +// limit"), which Fly served as 502s. Nothing warned first: the service had a +// disk watchdog, but disk was never the constraint (the archive was 68MB on a +// 1GB volume) while the heap was at 91%. +// +// Thresholds are lower than the disk watchdog's 80/95 because heap has no +// graceful degradation. Crossing the V8 limit is an immediate process abort, +// per-request allocation sits on top of the live set so the usable ceiling is +// below 100%, and recovery costs a full archive replay during which the service +// serves errors. Warn early enough to act. +// +// record_count is in the message on purpose: heap headroom divided by growth in +// records per day is the runway, and runway is what decides when the in-memory +// store has to become the disk-backed store that persistence.ts describes. +const HEAP_CHECK_INTERVAL_MS = 5 * 60 * 1000 // 5 minutes +let lastHeapAlert: HeapAlertLevel = 'ok' + +function heapUtilization(): { usedMb: number; limitMb: number; fraction: number } { + const stats = getHeapStatistics() + return { + usedMb: Math.round(stats.used_heap_size / 1024 / 1024), + limitMb: Math.round(stats.heap_size_limit / 1024 / 1024), + fraction: stats.used_heap_size / stats.heap_size_limit, + } +} + +function checkHeapUtilization(): void { + const { usedMb, limitMb, fraction } = heapUtilization() + const pct = (fraction * 100).toFixed(1) + const records = store.getRecordCount() + const level = heapAlertLevel(fraction) + + if (!shouldLogHeapTransition(lastHeapAlert, level)) return + + if (level === 'ok') { + // Reached only on a real transition, so this is a recovery from warn/error. + // eslint-disable-next-line no-console + console.log( + `atrib-graph: heap-watchdog recovered (${usedMb}MB/${limitMb}MB, ${pct}%, ${records} records)`, + ) + } else { + const threshold = level === 'error' ? HEAP_ERROR_FRACTION : HEAP_WARN_FRACTION + const msg = + `atrib-graph: heap-watchdog ${level.toUpperCase()}: ${usedMb}MB/${limitMb}MB ` + + `(${pct}%, ${records} records, threshold: ${(threshold * 100).toFixed(0)}%). ` + + `Raise --max-old-space-size and the fly.toml [[vm]] memory together, ` + + `or move to the disk-backed store described in src/persistence.ts.` + if (level === 'error') { + // eslint-disable-next-line no-console + console.error(msg) + } else { + // eslint-disable-next-line no-console + console.warn(msg) + } + } + lastHeapAlert = level +} + +{ + // One unconditional line per boot, so every restart leaves a heap datapoint + // in the logs even while utilization is healthy. Trending these across + // restarts is how the runway above gets measured. + const { usedMb, limitMb, fraction } = heapUtilization() + // eslint-disable-next-line no-console + console.log( + `atrib-graph: heap ${usedMb}MB/${limitMb}MB (${(fraction * 100).toFixed(1)}%) ` + + `at ${store.getRecordCount()} records`, + ) + const handle = setInterval(checkHeapUtilization, HEAP_CHECK_INTERVAL_MS) + handle.unref() +} + // Graceful shutdown async function shutdown(signal: string): Promise { // eslint-disable-next-line no-console diff --git a/services/graph-node/src/persistence.ts b/services/graph-node/src/persistence.ts index 60b3a7b7..c6884404 100644 --- a/services/graph-node/src/persistence.ts +++ b/services/graph-node/src/persistence.ts @@ -25,6 +25,13 @@ * per graph-node instance; beyond that the sustainable shape is a * disk-backed graph store (Layer 4 in the architecture roadmap), at * which point this archive becomes redundant and can be removed. + * + * The log passed 10^5 in July 2026, and the in-memory side of that boundary + * bit first: on 2026-07-29 the resident record set filled the V8 heap ceiling + * and graph-node began aborting on allocation. Disk was never the constraint + * (this archive was 68MB on a 1GB volume). Choosing the successor is tracked + * as P059 in DECISIONS.md; the heap watchdog in main.ts (src/heap-watchdog.ts) + * warns at 70% of the heap ceiling and that warning is P059's trigger. */ import { open, mkdir, stat } from 'node:fs/promises' diff --git a/services/graph-node/test/heap-watchdog.test.ts b/services/graph-node/test/heap-watchdog.test.ts new file mode 100644 index 00000000..8c228494 --- /dev/null +++ b/services/graph-node/test/heap-watchdog.test.ts @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: Apache-2.0 + +/** + * The heap watchdog exists because the 2026-07-29 outage gave no warning: the + * live heap reached the V8 ceiling at ~112k records and the process started + * aborting on allocation. These tests pin the thresholds, because an alerting + * path that never fires reads as an all-clear and is worse than no alert. + */ + +import { describe, it, expect } from 'vitest' +import { + HEAP_ERROR_FRACTION, + HEAP_WARN_FRACTION, + heapAlertLevel, + shouldLogHeapTransition, +} from '../src/heap-watchdog.js' + +describe('heapAlertLevel', () => { + it('is ok well below the warn threshold', () => { + // Where graph-node sat right after the 2026-07-29 fix: ~380MB of a 780MB + // ceiling. Healthy, and must not alert. + expect(heapAlertLevel(380 / 780)).toBe('ok') + }) + + it('warns exactly at the warn threshold and stays ok just below', () => { + expect(heapAlertLevel(HEAP_WARN_FRACTION)).toBe('warn') + expect(heapAlertLevel(HEAP_WARN_FRACTION - 0.001)).toBe('ok') + }) + + it('errors exactly at the error threshold and warns just below', () => { + expect(heapAlertLevel(HEAP_ERROR_FRACTION)).toBe('error') + expect(heapAlertLevel(HEAP_ERROR_FRACTION - 0.001)).toBe('warn') + }) + + it('errors at the utilization that actually took the service down', () => { + // The GC log showed ~480MB live against a 489MB ceiling before the abort. + expect(heapAlertLevel(480 / 489)).toBe('error') + }) + + it('fires before the ceiling, not at it', () => { + // The point of the watchdog: both thresholds must leave room to act. + expect(HEAP_WARN_FRACTION).toBeLessThan(HEAP_ERROR_FRACTION) + expect(HEAP_ERROR_FRACTION).toBeLessThan(1) + }) + + it('treats an unmeasurable heap as ok rather than alarming', () => { + expect(heapAlertLevel(Number.NaN)).toBe('ok') + expect(heapAlertLevel(Number.POSITIVE_INFINITY)).toBe('ok') + }) +}) + +describe('shouldLogHeapTransition', () => { + it('logs when crossing into warn and into error', () => { + expect(shouldLogHeapTransition('ok', 'warn')).toBe(true) + expect(shouldLogHeapTransition('warn', 'error')).toBe(true) + }) + + it('stays silent while a level holds, so steady state does not spam', () => { + expect(shouldLogHeapTransition('warn', 'warn')).toBe(false) + expect(shouldLogHeapTransition('error', 'error')).toBe(false) + expect(shouldLogHeapTransition('ok', 'ok')).toBe(false) + }) + + it('logs recovery back down', () => { + expect(shouldLogHeapTransition('error', 'warn')).toBe(true) + expect(shouldLogHeapTransition('warn', 'ok')).toBe(true) + }) +})