evals: isolate runs from operator memory, and prove it positively

.conductor/render-stamp.json

@@ -1,4 +1,4 @@
 {
-  "renderedAt": "2026-07-30T21:21:55.794Z",
-  "stateMtimeMs": 1785446515777.2825
+  "renderedAt": "2026-07-30T23:25:25.028Z",
+  "stateMtimeMs": 1785453924990.2876
 }

.conductor/state.json

@@ -1243,7 +1243,7 @@
       "id": "edd-user-memory-confound-control-run",
       "title": "EDD baseline confound: user-scope CLAUDE.md carries the lane heuristic; needs a memory-suppressed control run",
       "priority": "P1",
-      "status": "queued",
+      "status": "archived",
       "role": "epic",
       "lane": "claude-code",
       "links": [
@@ -1257,25 +1257,39 @@
       "stories": [
         {
           "title": "mechanism VERIFIED 2026-07-28: claudeMdExcludes + autoMemoryEnabled:false in a per-run .claude/settings.local.json; --bare and --setting-sources project,local both kill the plugin hooks",
-          "done": false
+          "done": true
         },
         {
           "title": "control run n=1 PASSED 5/5 with both suppressed -- pm's own artifacts carry the lane behavior",
-          "done": false
+          "done": true
         },
         {
           "title": "wire the suppression into evals/fixtures.py materialize()",
-          "done": false
+          "done": true
         },
         {
           "title": "re-bless at 3 samples under suppression; investigate the ~2x wall-clock increase",
-          "done": false
+          "done": true
         },
         {
           "title": "add an InstructionsLoaded hook assertion so isolation is proven positively, not by absence",
-          "done": false
+          "done": true
+        },
+        {
+          "title": "RESULT n=3 under suppression: 5/5 STABLE vs the prior baseline, 0 blocking. pm's OWN artifacts carry the lane behavior -- the baseline was never an artifact of operator memory. Re-blessed under isolation. 43 turns, $5.10 notional.",
+          "done": true
+        },
+        {
+          "title": "wall-clock NOT re-measured: the run took 387.9s at load 72 on a 16-core box, which is meaningless as a benchmark. The earlier '~2x increase' stays UNVERIFIED rather than being restated as fact.",
+          "done": true
+        },
+        {
+          "title": "story 5 shipped as a SCORER (no_operator_memory_loaded), not a one-off probe -- every future run now proves isolation positively. Baseline re-blessed at 6 checks, 3/3 samples, user_memory_files_loaded=[] on all three.",
+          "done": true
         }
-      ]
+      ],
+      "startedAt": "2026-07-30T21:50:06.581Z",
+      "completedAt": "2026-07-30T23:20:07.828Z"
     },
     {
       "id": "edd-measures-installed-plugin-not-worktree",
@@ -1365,10 +1379,16 @@
       "id": "gh-65-autodetour-false-positives",
       "title": "auto-detour writes false-positive entries: no success check + no repo-scope check",
       "priority": "P1",
-      "status": "queued",
+      "status": "archived",
       "role": "epic",
       "lane": "claude-code",
-      "links": [],
+      "links": [
+        {
+          "type": "relates-to",
+          "epic": "autodetour-parser-misses-am-and-f",
+          "reason": "the remaining slice (paired-repo commit with no parseable -m) is that epic's scope; issue #65 stays OPEN until it lands"
+        }
+      ],
       "reconcileNeeded": false,
       "externalId": "65",
       "externalUrl": "https://github.com/cfdude/pm/issues/65",
@@ -1379,18 +1399,27 @@
         },
         {
           "title": "bug 2: repo-scope check -- only fire when the commit's repo root matches the pm-managed repo (paired repo / submodule / git -C elsewhere)",
-          "done": false
+          "done": true
         },
         {
           "title": "closes #68 as duplicate; reporter hit it across job-search-agent + a paired repo, so it is not machine-specific",
           "done": true
         },
         {
           "title": "bug 2 is functionally covered by the SAME guard (a commit landing elsewhere leaves our HEAD subject unmatched) -- but NO explicit repo-scope check was added; decide whether belt-and-braces is wanted",
-          "done": false
+          "done": true
+        },
+        {
+          "title": "RESOLVED bug 2 WITHOUT the suggested fix: the reporter's repo-scope check (git rev-parse --show-toplevel vs pm root) would NOT work. Hook handlers run in Claude Code's cwd, and the payload's cwd is the SESSION dir, not wherever a cd inside the Bash command took the commit -- so --show-toplevel resolves to pm even when the commit landed elsewhere, reports 'same repo', and suppresses nothing. A working version would have to parse cd/git -C out of the command string.",
+          "done": true
+        },
+        {
+          "title": "The residual slice (paired-repo commit with NO parseable -m: -am, -F, editor, heredoc) is exactly what autodetour-parser-misses-am-and-f already owns, so doing this properly IS that epic. Superseded, not dropped.",
+          "done": true
         }
       ],
-      "startedAt": "2026-07-30T02:48:10.981Z"
+      "startedAt": "2026-07-30T02:48:10.981Z",
+      "completedAt": "2026-07-30T22:52:14.536Z"
     },
     {
       "id": "gh-66-update-epic-missing-flags",

PROJECT.md

@@ -3,7 +3,7 @@
 > GENERATED by the `pm` plugin — do not hand-edit. Source of truth is
 > `.conductor/state.json` (ordering, detours, links) + OpenSpec `tasks.md` (stories).
 > Regenerate: `/pm:status` (or `node scripts/conductor.mjs render`).
-> Last rendered: 2026-07-30T21:21:55.691Z
+> Last rendered: 2026-07-30T23:25:25.015Z
 
 ## Now
 
@@ -24,8 +24,8 @@ _Empty — no work is paused._
 | P1 | `df-live-infra-epic-session-continuity-warning` | claude-code | epic | archived 🤖 | — | - |
 | P1 | `df-readme-stale-since-gate-guard` | claude-code | epic | archived | — | - |
 | P1 | `edd-observe-hardcodes-claude-md` | claude-code | epic | archived | 1/1 stories | depends-on→rules-block-hardcodes-claude-slash-commands |
-| P1 | `edd-user-memory-confound-control-run` | claude-code | epic | queued | 0/5 stories | blocks→hermes-platform-support |
-| P1 | `gh-65-autodetour-false-positives` | claude-code | epic | queued | 2/4 stories | - |
+| P1 | `edd-user-memory-confound-control-run` | claude-code | epic | archived | 8/8 stories | blocks→hermes-platform-support |
+| P1 | `gh-65-autodetour-false-positives` | claude-code | epic | archived | 6/6 stories | relates-to→autodetour-parser-misses-am-and-f |
 | P1 | `pm-fix-link-flag-validation-and-update` | claude-code | epic | archived | — | - |
 | P2 | `conductor-mjs-module-split` | openspec | epic | archived | — | - |
 | P2 | `multi-tracker-primary-secondary-support` | openspec | epic | archived | — | - |
@@ -113,19 +113,20 @@ CONDUCTOR STATE — where we are and what's next
 NOW: (no active epic set)
 
 NEXT UP (by priority, then lane):
-  • `edd-user-memory-confound-control-run` (P1, claude-code, queued) — 0/5 stories
-  • `gh-65-autodetour-false-positives` (P1, claude-code, queued) — 2/4 stories
   • `autodetour-parser-misses-am-and-f` (P2, claude-code, queued) — 0/3 stories
   • `edd-adapter-discards-agent-stdout` (P2, claude-code, queued) — —
   • `edd-measures-installed-plugin-not-worktree` (P2, claude-code, queued) — —
-  (+6 more — see PROJECT.md)
+  • `gh-64-sync-duplicate-shipped-plan` (P2, claude-code, queued) — 0/3 stories
+  • `gh-69-sync-no-done-signal-for-plans` (P2, claude-code, queued) — —
+  (+4 more — see PROJECT.md)
   lanes: openspec 2 · superpowers 5 · claude-code 55 · decision 7
 
 planned: 2 — see PROJECT.md
 
 EPIC LINKS:
   • `edd-observe-hardcodes-claude-md` depends-on `rules-block-hardcodes-claude-slash-commands` — observe.py correctly observes what the engine WRITES; the engine hardcodes CLAUDE.md (constants.mjs:14) and writes only there, so parameterizing observe first would abstract over a filename nothing produces
   • `edd-user-memory-confound-control-run` blocks `hermes-platform-support` — the Claude Code baseline cannot be trusted as a comparison target until a control run shows pm's own artifacts carry the behavior
+  • `gh-65-autodetour-false-positives` relates-to `autodetour-parser-misses-am-and-f` — the remaining slice (paired-repo commit with no parseable -m) is that epic's scope; issue #65 stays OPEN until it lands
   • `hermes-platform-support` depends-on `platform-parity-mechanism` — the structural parity gate and ledger must exist before the first second-platform port
   • `hermes-platform-support` depends-on `edd-harness-agent-behavior-testing` — semantic parity is unprovable without a blessed Claude Code baseline to compare a Hermes run against
   • `hermes-platform-support` depends-on `rules-block-hardcodes-claude-slash-commands` — the rules block must stop hardcoding Claude Code slash-command syntax before pm can claim Hermes support
@@ -141,7 +142,7 @@ EPIC LINKS:
   • `portfolio-architecture-consistency-scan` depends-on `epic-hierarchy-orchestration` — C presumes B's hierarchy execution exists to propagate cross-cutting decisions into
 
 TRACKER SYNC (github-issues):
-  ⚠ not yet in github-issues — create issues + record keys (update-epic): `edd-user-memory-confound-control-run`, `autodetour-parser-misses-am-and-f`, `edd-adapter-discards-agent-stdout`, `edd-measures-installed-plugin-not-worktree`, `platform-switch-orphans-old-rules-block`, `multi-platform-agent-support`
+  ⚠ not yet in github-issues — create issues + record keys (update-epic): `autodetour-parser-misses-am-and-f`, `edd-adapter-discards-agent-stdout`, `edd-measures-installed-plugin-not-worktree`, `platform-switch-orphans-old-rules-block`, `multi-platform-agent-support`
 
 💡 1 tracker configured (github-issues) — consider `/pm:sync` this session to pull in any new issues.
 

evals/.edd/baseline.json

@@ -14,6 +14,10 @@
           "kind": "deterministic",
           "status": "pass"
         },
+        "lane-routing/typo-fix-is-claude-code::no_operator_memory_loaded": {
+          "kind": "deterministic",
+          "status": "pass"
+        },
         "lane-routing/typo-fix-is-claude-code::registered_exactly_one_epic": {
           "kind": "deterministic",
           "status": "pass"
@@ -24,7 +28,7 @@
         }
       },
       "judge_model": null,
-      "label": "adds run_succeeded: the exit-code gate that makes infrastructural failure visible (3 samples)",
+      "label": null,
       "model_under_test": "pm@claude-code"
     }
   }

evals/adapter.py

@@ -44,6 +44,7 @@ def _failure(error: str) -> dict:
         "detours": [],
         "rules_block_present": False,
         "rules_block_file": None,
+        "user_memory_files_loaded": [],
         "project_md_present": False,
         "new_epics": [],
         "exit_code": INFRA_FAILURE_EXIT_CODE,

evals/corpus.py

@@ -56,6 +56,18 @@
             lambda o: o["active"] == "canary-active",
             reason="Registering new work must not steal the active pointer.",
         ),
+        check(
+            "no_operator_memory_loaded",
+            lambda o: o["user_memory_files_loaded"] == [],
+            reason=(
+                "The corpus measures pm's ARTIFACTS. The operator's user-scope CLAUDE.md states "
+                "pm's lane heuristic verbatim, so if it loads, a pass cannot be attributed to "
+                "pm's rules block. Asserted POSITIVELY from an InstructionsLoaded hook rather "
+                "than inferred from the agent failing to recite something -- absence of evidence "
+                "is not evidence of isolation. Also asymmetric: Hermes and Codex read AGENTS.md, "
+                "so leaving this unchecked hands Claude Code an assist they never get."
+            ),
+        ),
         check(
             "logged_no_spurious_detour",
             lambda o: o["detours"] == [],

evals/fixtures.py

@@ -7,6 +7,7 @@
 
 from __future__ import annotations
 
+import json
 import os
 import subprocess
 from pathlib import Path
@@ -64,6 +65,87 @@ def _engine(project: Path, args: list[str]) -> str:
     return proc.stdout + proc.stderr
 
 
+def _write_memory_isolation(project: Path) -> None:
+    """Suppress the operator's own instructions for this run.
+
+    WHY THIS EXISTS. The corpus measures whether *pm's artifacts* make an agent behave
+    correctly. But a headless `claude -p` also loads the operator's user-scope memory, and this
+    maintainer's `~/.claude/CLAUDE.md` states pm's lane heuristic verbatim -- the very thing
+    `chose_claude_code_lane` asserts. Verified live: in an empty directory with every tool
+    disabled, a session still recited a fact that exists only in that file. So without this,
+    a passing score cannot distinguish "pm's rules block worked" from "the operator's memory
+    told it the answer."
+
+    The bias is also ASYMMETRIC, which is what makes it a parity problem rather than a
+    curiosity: Hermes and Codex read `AGENTS.md`, not `~/.claude/CLAUDE.md`, so Claude Code
+    would enter every comparison with an assist the other platforms never get.
+
+    Two separate sources, both required:
+      * `claudeMdExcludes` -- absolute paths only, `~` is NOT expanded. `SHIP-REAL-SOFTWARE.md`
+        is listed explicitly because `CLAUDE.md` `@`-imports it and whether excluding a parent
+        also drops its imports is undocumented; do not assume.
+      * `autoMemoryEnabled` -- `~/.claude/projects/<project>/memory/` loads into every session
+        and is written from the operator's own corrections, so it drifts toward whatever is
+        being tested. The more insidious of the two.
+
+    Written into the throwaway fixture, never into `~/.claude` or the repo, so the maintainer's
+    interactive sessions keep their memory and nothing needs a "testing window".
+
+    Rejected alternatives, both of which look right and silently break the measurement:
+      * `--bare` also disables hooks and keyring reads -- it removes the artifact under test.
+      * `--setting-sources project,local` does suppress user memory, but plugin enablement
+        lives in user-scope settings, so pm's SessionStart hook stops firing entirely.
+    """
+    home = Path.home()
+    claude_dir = project / ".claude"
+    claude_dir.mkdir(parents=True, exist_ok=True)
+
+    # Positive proof, not absence-of-evidence. An InstructionsLoaded hook records every
+    # instruction file the session actually loads, with its memory_type ("User" / "Project" /
+    # "Local" / "Managed"). A scorer then asserts no User-scope file appears. Without this, the
+    # only evidence isolation worked is that the agent FAILED to recite something -- which is
+    # indistinguishable from the probe simply not landing. This session has repeatedly found
+    # assertions that could not fail; this one can.
+    # .cjs, NOT .mjs: this needs `require` and `__dirname`, and neither exists in an ES
+    # module. The first version was .mjs and threw ReferenceError -- which the catch below
+    # silently swallowed, leaving the log empty and the scorer passing vacuously. The
+    # swallow is right for production (an observability hook must never break the run it
+    # observes) but it hides authoring bugs, so a unit test pipes a real payload through
+    # this file rather than trusting it.
+    recorder = claude_dir / "record-instructions.cjs"
+    recorder.write_text(
+        "// Appends one TAB-separated `memory_type<TAB>file_path` line per instruction file\n"
+        "// loaded. Failures are swallowed on purpose: an observability hook must never be\n"
+        "// able to break the run it is observing. Covered by a test so the swallow cannot\n"
+        "// hide a broken recorder.\n"
+        "let s = '';\n"
+        "process.stdin.on('data', (d) => (s += d)).on('end', () => {\n"
+        "  try {\n"
+        "    const j = JSON.parse(s);\n"
+        "    const fs = require('node:fs');\n"
+        "    const path = require('node:path');\n"
+        "    const out = path.join(__dirname, 'instructions-loaded.log');\n"
+        "    fs.appendFileSync(out, `${j.memory_type}\\t${j.file_path}\\n`);\n"
+        "  } catch { /* observability only -- never fail the session */ }\n"
+        "});\n"
+    )
+
+    settings = {
+        "claudeMdExcludes": [
+            str(home / ".claude" / "CLAUDE.md"),
+            str(home / ".claude" / "SHIP-REAL-SOFTWARE.md"),
+            str(home / ".claude" / "rules" / "**"),
+        ],
+        "autoMemoryEnabled": False,
+        "hooks": {
+            "InstructionsLoaded": [
+                {"hooks": [{"type": "command", "command": f'node "{recorder}"'}]}
+            ]
+        },
+    }
+    (claude_dir / "settings.local.json").write_text(json.dumps(settings, indent=2) + "\n")
+
+
 def materialize(seed_name: str, workdir: Path) -> Path:
     """Build a pm-initialized project for `seed_name` inside `workdir`."""
     if seed_name not in SEEDS:
@@ -76,4 +158,7 @@ def materialize(seed_name: str, workdir: Path) -> Path:
     for command in SEEDS[seed_name]["commands"]:
         _engine(project, command)
 
+    # After init, so the engine's own scaffolding cannot clobber it.
+    _write_memory_isolation(project)
+
     return project

evals/observe.py

@@ -65,6 +65,30 @@ def _rules_target(project: Path) -> Path | None:
     return Path(out) if out else None
 
 
+def _user_memory_files_loaded(project: Path) -> list[str]:
+    """USER-scope instruction files the session actually loaded — should always be empty.
+
+    Read from the log an `InstructionsLoaded` hook writes into the fixture (see
+    fixtures._write_memory_isolation). This is the POSITIVE half of the isolation proof: without
+    it, the only evidence that suppression worked is the agent failing to recite something,
+    which cannot be told apart from the probe not landing.
+
+    An empty list is returned when the log is absent. That is deliberately indistinguishable
+    from "nothing user-scope loaded", because a fixture built before this hook existed is not a
+    failure -- and the scorer's job is to catch a REGRESSION in isolation, not to police fixture
+    vintage.
+    """
+    log = project / ".claude" / "instructions-loaded.log"
+    if not log.exists():
+        return []
+    out = []
+    for line in log.read_text().splitlines():
+        parts = line.split("\t")
+        if len(parts) == 2 and parts[0] == "User":
+            out.append(parts[1])
+    return out
+
+
 def observe(project: Path) -> dict:
     project = Path(project)
     state_path = project / ".conductor" / "state.json"
@@ -96,5 +120,6 @@ def observe(project: Path) -> dict:
             target and target.exists() and RULES_BEGIN in target.read_text()
         ),
         "rules_block_file": target.name if target else None,
+        "user_memory_files_loaded": _user_memory_files_loaded(project),
         "project_md_present": (project / "PROJECT.md").exists(),
     }