ci: run the graph canary after both deploys settle

.github/workflows/deploy-services.yml

@@ -146,18 +146,11 @@ jobs:
         with:
           ref: ${{ github.event.workflow_run.head_sha || github.sha }}
 
-      - uses: pnpm/action-setup@v6
-
+      # No pnpm install / workspace build here: the graph canary moved to its
+      # own job below, and check-log-smoke.mjs imports only node:crypto.
       - uses: actions/setup-node@v7
         with:
           node-version: '24'
-          cache: pnpm
-
-      - name: Install canary dependencies
-        run: pnpm install --frozen-lockfile
-
-      - name: Build canary dependencies
-        run: pnpm --filter @atrib/mcp build
 
       - uses: superfly/flyctl-actions/setup-flyctl@ed8efb33836e8b2096c7fd3ba1c8afe303ebbff1
 
@@ -170,9 +163,6 @@ jobs:
         shell: bash
         run: node .github/scripts/check-log-smoke.mjs
 
-      - name: Smoke test log to graph indexing
-        run: pnpm --filter @atrib/log-node graph-canary
-
   deploy-graph-node:
     needs: changes
     if: needs.changes.outputs.graph_node == 'true'
@@ -189,6 +179,60 @@ jobs:
           FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
         run: flyctl deploy -c services/graph-node/fly.toml --remote-only
 
+  # The log-to-graph indexing canary runs in its own job so it starts AFTER
+  # both deploys settle rather than alongside them.
+  #
+  # It used to be the last step of deploy-log-node, which depends only on
+  # `changes`, so deploy-log-node and deploy-graph-node ran in parallel with no
+  # ordering. A commit touching a shared path (packages/mcp/*, package.json,
+  # pnpm-lock.yaml, pnpm-workspace.yaml, tsconfig.base.json, .dockerignore)
+  # sets both flags, and the canary could then poll graph-node while its
+  # machine was restarting into the new release. graph-node replays its whole
+  # record archive on start, so that window serves 502s and the canary fails on
+  # a system that is actually fine.
+  #
+  # It now also runs when only graph-node deployed. The canary exercises log
+  # submit through graph index, so a graph-node-only release is exactly the
+  # change it should gate; previously it ran on log-node releases only, which
+  # is why the 2026-07-29 trace regression shipped without the canary seeing it.
+  graph-canary:
+    needs: [changes, deploy-log-node, deploy-graph-node]
+    # !cancelled() keeps a skipped sibling deploy from skipping this job, since
+    # a skipped `needs` entry otherwise cascades, while still standing down when
+    # the run itself is cancelled. The guards then re-impose the real
+    # conditions: `changes` actually ran, at least one of the two services
+    # deployed, and neither deploy failed or was cancelled (a red canary after a
+    # failed deploy would only restate the deploy failure).
+    if: >-
+      !cancelled()
+      && needs.changes.result == 'success'
+      && (needs.changes.outputs.log_node == 'true' || needs.changes.outputs.graph_node == 'true')
+      && needs.deploy-log-node.result != 'failure'
+      && needs.deploy-graph-node.result != 'failure'
+      && needs.deploy-log-node.result != 'cancelled'
+      && needs.deploy-graph-node.result != 'cancelled'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          ref: ${{ github.event.workflow_run.head_sha || github.sha }}
+
+      - uses: pnpm/action-setup@v6
+
+      - uses: actions/setup-node@v7
+        with:
+          node-version: '24'
+          cache: pnpm
+
+      - name: Install canary dependencies
+        run: pnpm install --frozen-lockfile
+
+      - name: Build canary dependencies
+        run: pnpm --filter @atrib/mcp build
+
+      - name: Smoke test log to graph indexing
+        run: pnpm --filter @atrib/log-node graph-canary
+
   deploy-directory-node:
     needs: changes
     if: needs.changes.outputs.directory_node == 'true'

DECISIONS.md

@@ -10311,3 +10311,49 @@ pre-allocate.
 **Likely outcome (not committed):** accept at PR time; this entry exists so the ADR obligation is findable.
 
 **ADR number** will be assigned when the decision is acted on. Do not pre-allocate.
+
+## P059: graph-node's in-memory store needs a disk-backed successor before the heap ceiling returns
+
+**Filed 2026-07-29** after the graph-node outage fixed in
+[#599](https://github.com/creatornader/atrib/pull/599). That PR removed the
+trigger and raised the ceiling. It did not change the shape that produced the
+outage, so this entry exists so the shape is not forgotten between incidents.
+
+**Source:** graph-node holds every record and every derived index in memory. The
+live heap therefore grows linearly and without bound with the log. On 2026-07-29
+that live set reached the default V8 heap ceiling (`heap_size_limit` was 493MB on
+the 1024mb machine, because Node caps old-space near half a small machine's RAM
+regardless of the `[[vm]]` setting). Past that point the process aborted with
+"Ineffective mark-compacts near heap limit" on any further allocation, and Fly
+served the dead instance as 502s. `services/graph-node/src/persistence.ts` had
+already named the boundary: the in-memory shape is "sustainable until ~10^5
+records per graph-node instance; beyond that the sustainable shape is a
+disk-backed graph store." The log passed 10^5 in July 2026.
+
+**Measured position at filing.** 112,781 records, 439MB RSS, 780MB heap ceiling
+after the fix, so roughly half the ceiling consumed. Ingest was running about 4k
+to 7k records/day at roughly 3.4KB of heap each, which is 15MB to 22MB/day. That
+is a runway of weeks, not quarters. The heap watchdog added alongside this entry
+(`src/heap-watchdog.ts`, warn at 70%, error at 85%) exists to make the runway
+observable rather than inferred, and its warning is the intended trigger for
+acting on this entry.
+
+**The decision in question:** whether the successor is the disk-backed graph
+store persistence.ts anticipates, a bounded in-memory working set over the
+existing archive, sharding by `context_id` across instances, or simply
+continuing to raise the ceiling with machine size. The first three change the
+service's operational shape; the fourth is bounded by machine cost and only ever
+defers the same failure.
+
+**Not urgent as a correctness matter, and not open-ended either.** Whatever is
+chosen must preserve two properties the current design gets for free: [§3.2.4](atrib-spec.md#324-edge-derivation-rules)
+edge derivation stays deterministic over the full record set, and the [§1.9](atrib-spec.md#19-key-rotation-and-revocation)
+revocation registry stays a global scan, since a key revoked in one session
+retires it everywhere. A bounded working set is the option most in tension with
+both.
+
+**Likely outcome (not committed):** raise machine memory once more as a stopgap
+when the watchdog first warns, and treat that warning as the deadline for
+choosing among the three structural options rather than as the fix.
+
+**ADR number** will be assigned when the decision is acted on. Do not pre-allocate.

services/graph-node/fly.toml

@@ -42,9 +42,12 @@ primary_region = 'iad'
   # alone is not enough: see NODE_OPTIONS above, V8 will not use the extra RAM
   # unless told. Both settings move together.
   #
-  # Headroom is finite and the log only grows (~4k records/day at 2026-07-29,
-  # ~4KB each in heap, so ~17MB/day). At a 768MB heap that is a few weeks
-  # before this becomes tight again. The durable fix is the disk-backed graph
-  # store already flagged in src/persistence.ts, which notes this in-memory
-  # shape is sustainable to ~10^5 records; the log passed that in July 2026.
+  # Headroom is finite and the log only grows (~4k to 7k records/day at
+  # 2026-07-29, ~3.4KB each in heap, so ~15-22MB/day). At a 768MB heap that is
+  # weeks, not quarters. The durable fix is the disk-backed graph store already
+  # flagged in src/persistence.ts, which notes this in-memory shape is
+  # sustainable to ~10^5 records; the log passed that in July 2026. Choosing
+  # the successor is tracked as P059 in DECISIONS.md, and the heap watchdog
+  # (src/heap-watchdog.ts) warns at 70% so the runway is observed, not guessed.
+  # If you raise this, raise NODE_OPTIONS above with it or nothing changes.
   memory = '1024mb'

services/graph-node/src/heap-watchdog.ts

@@ -0,0 +1,54 @@
+// SPDX-License-Identifier: Apache-2.0
+
+/**
+ * Threshold logic for the heap-utilization watchdog in main.ts.
+ *
+ * Lives in its own module so the thresholds are unit-testable. The watchdog's
+ * whole value is firing at the right moment; an alerting path that quietly
+ * never fires is worse than no alert, because it reads as an all-clear.
+ *
+ * Context: graph-node holds every record and derived index in memory, so the
+ * live heap grows linearly and without bound with the log. On 2026-07-29 that
+ * live set reached the V8 ceiling at ~112k records and the process began
+ * aborting on any further allocation, which Fly served as 502s. Nothing warned
+ * first.
+ *
+ * Thresholds sit below the disk watchdog's 80/95 because heap has no graceful
+ * degradation: crossing the V8 limit aborts the process immediately,
+ * per-request allocation sits on top of the live set so the usable ceiling is
+ * under 100%, and recovery costs a full archive replay that serves errors
+ * throughout.
+ */
+
+export const HEAP_WARN_FRACTION = 0.7
+export const HEAP_ERROR_FRACTION = 0.85
+
+export type HeapAlertLevel = 'ok' | 'warn' | 'error'
+
+/**
+ * Classify a heap-utilization fraction (used / limit).
+ *
+ * Non-finite input (a zero limit, or NaN from a runtime that does not report
+ * heap statistics) classifies as 'ok': the watchdog is advisory and must never
+ * turn a missing measurement into a false alarm.
+ */
+export function heapAlertLevel(fraction: number): HeapAlertLevel {
+  if (!Number.isFinite(fraction)) return 'ok'
+  if (fraction >= HEAP_ERROR_FRACTION) return 'error'
+  if (fraction >= HEAP_WARN_FRACTION) return 'warn'
+  return 'ok'
+}
+
+/**
+ * Whether a transition from `previous` to `current` should be logged.
+ *
+ * Only transitions are logged: a steady warn state would otherwise repeat every
+ * interval and train operators to filter the message out. Recovery to 'ok' is
+ * logged so the all-clear is visible, but only when something was wrong before.
+ */
+export function shouldLogHeapTransition(
+  previous: HeapAlertLevel,
+  current: HeapAlertLevel,
+): boolean {
+  return current !== previous
+}

services/graph-node/src/main.ts

@@ -27,6 +27,14 @@ import { createRecordStore } from './store.js'
 import { createArchiveAppender, replayArchive } from './persistence.js'
 import { statfs } from 'node:fs/promises'
 import { dirname } from 'node:path'
+import { getHeapStatistics } from 'node:v8'
+import {
+  HEAP_ERROR_FRACTION,
+  HEAP_WARN_FRACTION,
+  heapAlertLevel,
+  shouldLogHeapTransition,
+  type HeapAlertLevel,
+} from './heap-watchdog.js'
 import type { AtribRecord } from '@atrib/mcp'
 
 const port = parseInt(process.env.PORT ?? '3200', 10)
@@ -124,6 +132,84 @@ if (archivePath) {
   handle.unref()
 }
 
+// Periodic heap-utilization watchdog. Mirrors the disk watchdog above, for the
+// resource that actually takes this service down.
+//
+// graph-node keeps every record and every derived index in memory, so the live
+// heap grows linearly and without bound as the log grows. On 2026-07-29 that
+// live set reached the V8 heap ceiling at ~112k records and the process began
+// aborting on any further allocation ("Ineffective mark-compacts near heap
+// limit"), which Fly served as 502s. Nothing warned first: the service had a
+// disk watchdog, but disk was never the constraint (the archive was 68MB on a
+// 1GB volume) while the heap was at 91%.
+//
+// Thresholds are lower than the disk watchdog's 80/95 because heap has no
+// graceful degradation. Crossing the V8 limit is an immediate process abort,
+// per-request allocation sits on top of the live set so the usable ceiling is
+// below 100%, and recovery costs a full archive replay during which the service
+// serves errors. Warn early enough to act.
+//
+// record_count is in the message on purpose: heap headroom divided by growth in
+// records per day is the runway, and runway is what decides when the in-memory
+// store has to become the disk-backed store that persistence.ts describes.
+const HEAP_CHECK_INTERVAL_MS = 5 * 60 * 1000 // 5 minutes
+let lastHeapAlert: HeapAlertLevel = 'ok'
+
+function heapUtilization(): { usedMb: number; limitMb: number; fraction: number } {
+  const stats = getHeapStatistics()
+  return {
+    usedMb: Math.round(stats.used_heap_size / 1024 / 1024),
+    limitMb: Math.round(stats.heap_size_limit / 1024 / 1024),
+    fraction: stats.used_heap_size / stats.heap_size_limit,
+  }
+}
+
+function checkHeapUtilization(): void {
+  const { usedMb, limitMb, fraction } = heapUtilization()
+  const pct = (fraction * 100).toFixed(1)
+  const records = store.getRecordCount()
+  const level = heapAlertLevel(fraction)
+
+  if (!shouldLogHeapTransition(lastHeapAlert, level)) return
+
+  if (level === 'ok') {
+    // Reached only on a real transition, so this is a recovery from warn/error.
+    // eslint-disable-next-line no-console
+    console.log(
+      `atrib-graph: heap-watchdog recovered (${usedMb}MB/${limitMb}MB, ${pct}%, ${records} records)`,
+    )
+  } else {
+    const threshold = level === 'error' ? HEAP_ERROR_FRACTION : HEAP_WARN_FRACTION
+    const msg =
+      `atrib-graph: heap-watchdog ${level.toUpperCase()}: ${usedMb}MB/${limitMb}MB ` +
+      `(${pct}%, ${records} records, threshold: ${(threshold * 100).toFixed(0)}%). ` +
+      `Raise --max-old-space-size and the fly.toml [[vm]] memory together, ` +
+      `or move to the disk-backed store described in src/persistence.ts.`
+    if (level === 'error') {
+      // eslint-disable-next-line no-console
+      console.error(msg)
+    } else {
+      // eslint-disable-next-line no-console
+      console.warn(msg)
+    }
+  }
+  lastHeapAlert = level
+}
+
+{
+  // One unconditional line per boot, so every restart leaves a heap datapoint
+  // in the logs even while utilization is healthy. Trending these across
+  // restarts is how the runway above gets measured.
+  const { usedMb, limitMb, fraction } = heapUtilization()
+  // eslint-disable-next-line no-console
+  console.log(
+    `atrib-graph: heap ${usedMb}MB/${limitMb}MB (${(fraction * 100).toFixed(1)}%) ` +
+      `at ${store.getRecordCount()} records`,
+  )
+  const handle = setInterval(checkHeapUtilization, HEAP_CHECK_INTERVAL_MS)
+  handle.unref()
+}
+
 // Graceful shutdown
 async function shutdown(signal: string): Promise<void> {
   // eslint-disable-next-line no-console

services/graph-node/src/persistence.ts

@@ -25,6 +25,13 @@
  * per graph-node instance; beyond that the sustainable shape is a
  * disk-backed graph store (Layer 4 in the architecture roadmap), at
  * which point this archive becomes redundant and can be removed.
+ *
+ * The log passed 10^5 in July 2026, and the in-memory side of that boundary
+ * bit first: on 2026-07-29 the resident record set filled the V8 heap ceiling
+ * and graph-node began aborting on allocation. Disk was never the constraint
+ * (this archive was 68MB on a 1GB volume). Choosing the successor is tracked
+ * as P059 in DECISIONS.md; the heap watchdog in main.ts (src/heap-watchdog.ts)
+ * warns at 70% of the heap ceiling and that warning is P059's trigger.
  */
 
 import { open, mkdir, stat } from 'node:fs/promises'

services/graph-node/test/heap-watchdog.test.ts

@@ -0,0 +1,68 @@
+// SPDX-License-Identifier: Apache-2.0
+
+/**
+ * The heap watchdog exists because the 2026-07-29 outage gave no warning: the
+ * live heap reached the V8 ceiling at ~112k records and the process started
+ * aborting on allocation. These tests pin the thresholds, because an alerting
+ * path that never fires reads as an all-clear and is worse than no alert.
+ */
+
+import { describe, it, expect } from 'vitest'
+import {
+  HEAP_ERROR_FRACTION,
+  HEAP_WARN_FRACTION,
+  heapAlertLevel,
+  shouldLogHeapTransition,
+} from '../src/heap-watchdog.js'
+
+describe('heapAlertLevel', () => {
+  it('is ok well below the warn threshold', () => {
+    // Where graph-node sat right after the 2026-07-29 fix: ~380MB of a 780MB
+    // ceiling. Healthy, and must not alert.
+    expect(heapAlertLevel(380 / 780)).toBe('ok')
+  })
+
+  it('warns exactly at the warn threshold and stays ok just below', () => {
+    expect(heapAlertLevel(HEAP_WARN_FRACTION)).toBe('warn')
+    expect(heapAlertLevel(HEAP_WARN_FRACTION - 0.001)).toBe('ok')
+  })
+
+  it('errors exactly at the error threshold and warns just below', () => {
+    expect(heapAlertLevel(HEAP_ERROR_FRACTION)).toBe('error')
+    expect(heapAlertLevel(HEAP_ERROR_FRACTION - 0.001)).toBe('warn')
+  })
+
+  it('errors at the utilization that actually took the service down', () => {
+    // The GC log showed ~480MB live against a 489MB ceiling before the abort.
+    expect(heapAlertLevel(480 / 489)).toBe('error')
+  })
+
+  it('fires before the ceiling, not at it', () => {
+    // The point of the watchdog: both thresholds must leave room to act.
+    expect(HEAP_WARN_FRACTION).toBeLessThan(HEAP_ERROR_FRACTION)
+    expect(HEAP_ERROR_FRACTION).toBeLessThan(1)
+  })
+
+  it('treats an unmeasurable heap as ok rather than alarming', () => {
+    expect(heapAlertLevel(Number.NaN)).toBe('ok')
+    expect(heapAlertLevel(Number.POSITIVE_INFINITY)).toBe('ok')
+  })
+})
+
+describe('shouldLogHeapTransition', () => {
+  it('logs when crossing into warn and into error', () => {
+    expect(shouldLogHeapTransition('ok', 'warn')).toBe(true)
+    expect(shouldLogHeapTransition('warn', 'error')).toBe(true)
+  })
+
+  it('stays silent while a level holds, so steady state does not spam', () => {
+    expect(shouldLogHeapTransition('warn', 'warn')).toBe(false)
+    expect(shouldLogHeapTransition('error', 'error')).toBe(false)
+    expect(shouldLogHeapTransition('ok', 'ok')).toBe(false)
+  })
+
+  it('logs recovery back down', () => {
+    expect(shouldLogHeapTransition('error', 'warn')).toBe(true)
+    expect(shouldLogHeapTransition('warn', 'ok')).toBe(true)
+  })
+})