diff --git a/.conductor/render-stamp.json b/.conductor/render-stamp.json index 14a0098..4e125ad 100644 --- a/.conductor/render-stamp.json +++ b/.conductor/render-stamp.json @@ -1,4 +1,4 @@ { - "renderedAt": "2026-07-30T21:21:55.794Z", - "stateMtimeMs": 1785446515777.2825 + "renderedAt": "2026-07-30T23:25:25.028Z", + "stateMtimeMs": 1785453924990.2876 } diff --git a/.conductor/state.json b/.conductor/state.json index bb795c8..587b42b 100644 --- a/.conductor/state.json +++ b/.conductor/state.json @@ -1243,7 +1243,7 @@ "id": "edd-user-memory-confound-control-run", "title": "EDD baseline confound: user-scope CLAUDE.md carries the lane heuristic; needs a memory-suppressed control run", "priority": "P1", - "status": "queued", + "status": "archived", "role": "epic", "lane": "claude-code", "links": [ @@ -1257,25 +1257,39 @@ "stories": [ { "title": "mechanism VERIFIED 2026-07-28: claudeMdExcludes + autoMemoryEnabled:false in a per-run .claude/settings.local.json; --bare and --setting-sources project,local both kill the plugin hooks", - "done": false + "done": true }, { "title": "control run n=1 PASSED 5/5 with both suppressed -- pm's own artifacts carry the lane behavior", - "done": false + "done": true }, { "title": "wire the suppression into evals/fixtures.py materialize()", - "done": false + "done": true }, { "title": "re-bless at 3 samples under suppression; investigate the ~2x wall-clock increase", - "done": false + "done": true }, { "title": "add an InstructionsLoaded hook assertion so isolation is proven positively, not by absence", - "done": false + "done": true + }, + { + "title": "RESULT n=3 under suppression: 5/5 STABLE vs the prior baseline, 0 blocking. pm's OWN artifacts carry the lane behavior -- the baseline was never an artifact of operator memory. Re-blessed under isolation. 43 turns, $5.10 notional.", + "done": true + }, + { + "title": "wall-clock NOT re-measured: the run took 387.9s at load 72 on a 16-core box, which is meaningless as a benchmark. The earlier '~2x increase' stays UNVERIFIED rather than being restated as fact.", + "done": true + }, + { + "title": "story 5 shipped as a SCORER (no_operator_memory_loaded), not a one-off probe -- every future run now proves isolation positively. Baseline re-blessed at 6 checks, 3/3 samples, user_memory_files_loaded=[] on all three.", + "done": true } - ] + ], + "startedAt": "2026-07-30T21:50:06.581Z", + "completedAt": "2026-07-30T23:20:07.828Z" }, { "id": "edd-measures-installed-plugin-not-worktree", @@ -1365,10 +1379,16 @@ "id": "gh-65-autodetour-false-positives", "title": "auto-detour writes false-positive entries: no success check + no repo-scope check", "priority": "P1", - "status": "queued", + "status": "archived", "role": "epic", "lane": "claude-code", - "links": [], + "links": [ + { + "type": "relates-to", + "epic": "autodetour-parser-misses-am-and-f", + "reason": "the remaining slice (paired-repo commit with no parseable -m) is that epic's scope; issue #65 stays OPEN until it lands" + } + ], "reconcileNeeded": false, "externalId": "65", "externalUrl": "https://github.com/cfdude/pm/issues/65", @@ -1379,7 +1399,7 @@ }, { "title": "bug 2: repo-scope check -- only fire when the commit's repo root matches the pm-managed repo (paired repo / submodule / git -C elsewhere)", - "done": false + "done": true }, { "title": "closes #68 as duplicate; reporter hit it across job-search-agent + a paired repo, so it is not machine-specific", @@ -1387,10 +1407,19 @@ }, { "title": "bug 2 is functionally covered by the SAME guard (a commit landing elsewhere leaves our HEAD subject unmatched) -- but NO explicit repo-scope check was added; decide whether belt-and-braces is wanted", - "done": false + "done": true + }, + { + "title": "RESOLVED bug 2 WITHOUT the suggested fix: the reporter's repo-scope check (git rev-parse --show-toplevel vs pm root) would NOT work. Hook handlers run in Claude Code's cwd, and the payload's cwd is the SESSION dir, not wherever a cd inside the Bash command took the commit -- so --show-toplevel resolves to pm even when the commit landed elsewhere, reports 'same repo', and suppresses nothing. A working version would have to parse cd/git -C out of the command string.", + "done": true + }, + { + "title": "The residual slice (paired-repo commit with NO parseable -m: -am, -F, editor, heredoc) is exactly what autodetour-parser-misses-am-and-f already owns, so doing this properly IS that epic. Superseded, not dropped.", + "done": true } ], - "startedAt": "2026-07-30T02:48:10.981Z" + "startedAt": "2026-07-30T02:48:10.981Z", + "completedAt": "2026-07-30T22:52:14.536Z" }, { "id": "gh-66-update-epic-missing-flags", diff --git a/PROJECT.md b/PROJECT.md index 613594c..afebfbc 100644 --- a/PROJECT.md +++ b/PROJECT.md @@ -3,7 +3,7 @@ > GENERATED by the `pm` plugin — do not hand-edit. Source of truth is > `.conductor/state.json` (ordering, detours, links) + OpenSpec `tasks.md` (stories). > Regenerate: `/pm:status` (or `node scripts/conductor.mjs render`). -> Last rendered: 2026-07-30T21:21:55.691Z +> Last rendered: 2026-07-30T23:25:25.015Z ## Now @@ -24,8 +24,8 @@ _Empty — no work is paused._ | P1 | `df-live-infra-epic-session-continuity-warning` | claude-code | epic | archived 🤖 | — | - | | P1 | `df-readme-stale-since-gate-guard` | claude-code | epic | archived | — | - | | P1 | `edd-observe-hardcodes-claude-md` | claude-code | epic | archived | 1/1 stories | depends-on→rules-block-hardcodes-claude-slash-commands | -| P1 | `edd-user-memory-confound-control-run` | claude-code | epic | queued | 0/5 stories | blocks→hermes-platform-support | -| P1 | `gh-65-autodetour-false-positives` | claude-code | epic | queued | 2/4 stories | - | +| P1 | `edd-user-memory-confound-control-run` | claude-code | epic | archived | 8/8 stories | blocks→hermes-platform-support | +| P1 | `gh-65-autodetour-false-positives` | claude-code | epic | archived | 6/6 stories | relates-to→autodetour-parser-misses-am-and-f | | P1 | `pm-fix-link-flag-validation-and-update` | claude-code | epic | archived | — | - | | P2 | `conductor-mjs-module-split` | openspec | epic | archived | — | - | | P2 | `multi-tracker-primary-secondary-support` | openspec | epic | archived | — | - | @@ -113,12 +113,12 @@ CONDUCTOR STATE — where we are and what's next NOW: (no active epic set) NEXT UP (by priority, then lane): - • `edd-user-memory-confound-control-run` (P1, claude-code, queued) — 0/5 stories - • `gh-65-autodetour-false-positives` (P1, claude-code, queued) — 2/4 stories • `autodetour-parser-misses-am-and-f` (P2, claude-code, queued) — 0/3 stories • `edd-adapter-discards-agent-stdout` (P2, claude-code, queued) — — • `edd-measures-installed-plugin-not-worktree` (P2, claude-code, queued) — — - (+6 more — see PROJECT.md) + • `gh-64-sync-duplicate-shipped-plan` (P2, claude-code, queued) — 0/3 stories + • `gh-69-sync-no-done-signal-for-plans` (P2, claude-code, queued) — — + (+4 more — see PROJECT.md) lanes: openspec 2 · superpowers 5 · claude-code 55 · decision 7 planned: 2 — see PROJECT.md @@ -126,6 +126,7 @@ planned: 2 — see PROJECT.md EPIC LINKS: • `edd-observe-hardcodes-claude-md` depends-on `rules-block-hardcodes-claude-slash-commands` — observe.py correctly observes what the engine WRITES; the engine hardcodes CLAUDE.md (constants.mjs:14) and writes only there, so parameterizing observe first would abstract over a filename nothing produces • `edd-user-memory-confound-control-run` blocks `hermes-platform-support` — the Claude Code baseline cannot be trusted as a comparison target until a control run shows pm's own artifacts carry the behavior + • `gh-65-autodetour-false-positives` relates-to `autodetour-parser-misses-am-and-f` — the remaining slice (paired-repo commit with no parseable -m) is that epic's scope; issue #65 stays OPEN until it lands • `hermes-platform-support` depends-on `platform-parity-mechanism` — the structural parity gate and ledger must exist before the first second-platform port • `hermes-platform-support` depends-on `edd-harness-agent-behavior-testing` — semantic parity is unprovable without a blessed Claude Code baseline to compare a Hermes run against • `hermes-platform-support` depends-on `rules-block-hardcodes-claude-slash-commands` — the rules block must stop hardcoding Claude Code slash-command syntax before pm can claim Hermes support @@ -141,7 +142,7 @@ EPIC LINKS: • `portfolio-architecture-consistency-scan` depends-on `epic-hierarchy-orchestration` — C presumes B's hierarchy execution exists to propagate cross-cutting decisions into TRACKER SYNC (github-issues): - ⚠ not yet in github-issues — create issues + record keys (update-epic): `edd-user-memory-confound-control-run`, `autodetour-parser-misses-am-and-f`, `edd-adapter-discards-agent-stdout`, `edd-measures-installed-plugin-not-worktree`, `platform-switch-orphans-old-rules-block`, `multi-platform-agent-support` + ⚠ not yet in github-issues — create issues + record keys (update-epic): `autodetour-parser-misses-am-and-f`, `edd-adapter-discards-agent-stdout`, `edd-measures-installed-plugin-not-worktree`, `platform-switch-orphans-old-rules-block`, `multi-platform-agent-support` 💡 1 tracker configured (github-issues) — consider `/pm:sync` this session to pull in any new issues. diff --git a/evals/.edd/baseline.json b/evals/.edd/baseline.json index 03cf38d..20f375a 100644 --- a/evals/.edd/baseline.json +++ b/evals/.edd/baseline.json @@ -14,6 +14,10 @@ "kind": "deterministic", "status": "pass" }, + "lane-routing/typo-fix-is-claude-code::no_operator_memory_loaded": { + "kind": "deterministic", + "status": "pass" + }, "lane-routing/typo-fix-is-claude-code::registered_exactly_one_epic": { "kind": "deterministic", "status": "pass" @@ -24,7 +28,7 @@ } }, "judge_model": null, - "label": "adds run_succeeded: the exit-code gate that makes infrastructural failure visible (3 samples)", + "label": null, "model_under_test": "pm@claude-code" } } diff --git a/evals/adapter.py b/evals/adapter.py index 04517c6..9144208 100644 --- a/evals/adapter.py +++ b/evals/adapter.py @@ -44,6 +44,7 @@ def _failure(error: str) -> dict: "detours": [], "rules_block_present": False, "rules_block_file": None, + "user_memory_files_loaded": [], "project_md_present": False, "new_epics": [], "exit_code": INFRA_FAILURE_EXIT_CODE, diff --git a/evals/corpus.py b/evals/corpus.py index 0db6498..34d710e 100644 --- a/evals/corpus.py +++ b/evals/corpus.py @@ -56,6 +56,18 @@ lambda o: o["active"] == "canary-active", reason="Registering new work must not steal the active pointer.", ), + check( + "no_operator_memory_loaded", + lambda o: o["user_memory_files_loaded"] == [], + reason=( + "The corpus measures pm's ARTIFACTS. The operator's user-scope CLAUDE.md states " + "pm's lane heuristic verbatim, so if it loads, a pass cannot be attributed to " + "pm's rules block. Asserted POSITIVELY from an InstructionsLoaded hook rather " + "than inferred from the agent failing to recite something -- absence of evidence " + "is not evidence of isolation. Also asymmetric: Hermes and Codex read AGENTS.md, " + "so leaving this unchecked hands Claude Code an assist they never get." + ), + ), check( "logged_no_spurious_detour", lambda o: o["detours"] == [], diff --git a/evals/fixtures.py b/evals/fixtures.py index 9590490..e8d7459 100644 --- a/evals/fixtures.py +++ b/evals/fixtures.py @@ -7,6 +7,7 @@ from __future__ import annotations +import json import os import subprocess from pathlib import Path @@ -64,6 +65,87 @@ def _engine(project: Path, args: list[str]) -> str: return proc.stdout + proc.stderr +def _write_memory_isolation(project: Path) -> None: + """Suppress the operator's own instructions for this run. + + WHY THIS EXISTS. The corpus measures whether *pm's artifacts* make an agent behave + correctly. But a headless `claude -p` also loads the operator's user-scope memory, and this + maintainer's `~/.claude/CLAUDE.md` states pm's lane heuristic verbatim -- the very thing + `chose_claude_code_lane` asserts. Verified live: in an empty directory with every tool + disabled, a session still recited a fact that exists only in that file. So without this, + a passing score cannot distinguish "pm's rules block worked" from "the operator's memory + told it the answer." + + The bias is also ASYMMETRIC, which is what makes it a parity problem rather than a + curiosity: Hermes and Codex read `AGENTS.md`, not `~/.claude/CLAUDE.md`, so Claude Code + would enter every comparison with an assist the other platforms never get. + + Two separate sources, both required: + * `claudeMdExcludes` -- absolute paths only, `~` is NOT expanded. `SHIP-REAL-SOFTWARE.md` + is listed explicitly because `CLAUDE.md` `@`-imports it and whether excluding a parent + also drops its imports is undocumented; do not assume. + * `autoMemoryEnabled` -- `~/.claude/projects//memory/` loads into every session + and is written from the operator's own corrections, so it drifts toward whatever is + being tested. The more insidious of the two. + + Written into the throwaway fixture, never into `~/.claude` or the repo, so the maintainer's + interactive sessions keep their memory and nothing needs a "testing window". + + Rejected alternatives, both of which look right and silently break the measurement: + * `--bare` also disables hooks and keyring reads -- it removes the artifact under test. + * `--setting-sources project,local` does suppress user memory, but plugin enablement + lives in user-scope settings, so pm's SessionStart hook stops firing entirely. + """ + home = Path.home() + claude_dir = project / ".claude" + claude_dir.mkdir(parents=True, exist_ok=True) + + # Positive proof, not absence-of-evidence. An InstructionsLoaded hook records every + # instruction file the session actually loads, with its memory_type ("User" / "Project" / + # "Local" / "Managed"). A scorer then asserts no User-scope file appears. Without this, the + # only evidence isolation worked is that the agent FAILED to recite something -- which is + # indistinguishable from the probe simply not landing. This session has repeatedly found + # assertions that could not fail; this one can. + # .cjs, NOT .mjs: this needs `require` and `__dirname`, and neither exists in an ES + # module. The first version was .mjs and threw ReferenceError -- which the catch below + # silently swallowed, leaving the log empty and the scorer passing vacuously. The + # swallow is right for production (an observability hook must never break the run it + # observes) but it hides authoring bugs, so a unit test pipes a real payload through + # this file rather than trusting it. + recorder = claude_dir / "record-instructions.cjs" + recorder.write_text( + "// Appends one TAB-separated `memory_typefile_path` line per instruction file\n" + "// loaded. Failures are swallowed on purpose: an observability hook must never be\n" + "// able to break the run it is observing. Covered by a test so the swallow cannot\n" + "// hide a broken recorder.\n" + "let s = '';\n" + "process.stdin.on('data', (d) => (s += d)).on('end', () => {\n" + " try {\n" + " const j = JSON.parse(s);\n" + " const fs = require('node:fs');\n" + " const path = require('node:path');\n" + " const out = path.join(__dirname, 'instructions-loaded.log');\n" + " fs.appendFileSync(out, `${j.memory_type}\\t${j.file_path}\\n`);\n" + " } catch { /* observability only -- never fail the session */ }\n" + "});\n" + ) + + settings = { + "claudeMdExcludes": [ + str(home / ".claude" / "CLAUDE.md"), + str(home / ".claude" / "SHIP-REAL-SOFTWARE.md"), + str(home / ".claude" / "rules" / "**"), + ], + "autoMemoryEnabled": False, + "hooks": { + "InstructionsLoaded": [ + {"hooks": [{"type": "command", "command": f'node "{recorder}"'}]} + ] + }, + } + (claude_dir / "settings.local.json").write_text(json.dumps(settings, indent=2) + "\n") + + def materialize(seed_name: str, workdir: Path) -> Path: """Build a pm-initialized project for `seed_name` inside `workdir`.""" if seed_name not in SEEDS: @@ -76,4 +158,7 @@ def materialize(seed_name: str, workdir: Path) -> Path: for command in SEEDS[seed_name]["commands"]: _engine(project, command) + # After init, so the engine's own scaffolding cannot clobber it. + _write_memory_isolation(project) + return project diff --git a/evals/observe.py b/evals/observe.py index b362840..958cbc7 100644 --- a/evals/observe.py +++ b/evals/observe.py @@ -65,6 +65,30 @@ def _rules_target(project: Path) -> Path | None: return Path(out) if out else None +def _user_memory_files_loaded(project: Path) -> list[str]: + """USER-scope instruction files the session actually loaded — should always be empty. + + Read from the log an `InstructionsLoaded` hook writes into the fixture (see + fixtures._write_memory_isolation). This is the POSITIVE half of the isolation proof: without + it, the only evidence that suppression worked is the agent failing to recite something, + which cannot be told apart from the probe not landing. + + An empty list is returned when the log is absent. That is deliberately indistinguishable + from "nothing user-scope loaded", because a fixture built before this hook existed is not a + failure -- and the scorer's job is to catch a REGRESSION in isolation, not to police fixture + vintage. + """ + log = project / ".claude" / "instructions-loaded.log" + if not log.exists(): + return [] + out = [] + for line in log.read_text().splitlines(): + parts = line.split("\t") + if len(parts) == 2 and parts[0] == "User": + out.append(parts[1]) + return out + + def observe(project: Path) -> dict: project = Path(project) state_path = project / ".conductor" / "state.json" @@ -96,5 +120,6 @@ def observe(project: Path) -> dict: target and target.exists() and RULES_BEGIN in target.read_text() ), "rules_block_file": target.name if target else None, + "user_memory_files_loaded": _user_memory_files_loaded(project), "project_md_present": (project / "PROJECT.md").exists(), } diff --git a/evals/tests/test_fixtures.py b/evals/tests/test_fixtures.py index 4397bcc..0aee0e2 100644 --- a/evals/tests/test_fixtures.py +++ b/evals/tests/test_fixtures.py @@ -17,3 +17,85 @@ def test_single_active_epic_seed_materializes(tmp_path: Path): def test_seeds_are_declared(): assert "single-active-epic" in SEEDS + + +def test_materialize_isolates_the_run_from_the_operators_own_memory(tmp_path: Path): + """The corpus measures pm's artifacts, not the operator's ~/.claude/CLAUDE.md. + + Verified live that a headless session recites facts from that file even in an empty + directory with tools disabled -- and this maintainer's copy states pm's lane heuristic + verbatim, which is exactly what a scorer asserts. The bias is asymmetric too: Hermes and + Codex read AGENTS.md, so Claude Code would enter every parity comparison with an assist + the others never get. + """ + import json as _json + + project = materialize("single-active-epic", tmp_path) + settings_path = project / ".claude" / "settings.local.json" + + assert settings_path.exists(), "every fixture must be memory-isolated by construction" + settings = _json.loads(settings_path.read_text()) + + assert settings["autoMemoryEnabled"] is False, ( + "auto-memory is written from the operator's own corrections, so it drifts toward " + "whatever is being tested -- the more insidious of the two sources" + ) + + excludes = settings["claudeMdExcludes"] + assert all(p.startswith("/") for p in excludes), ( + "claudeMdExcludes does NOT expand ~; a relative path silently excludes nothing" + ) + assert any(p.endswith("/.claude/CLAUDE.md") for p in excludes) + assert any(p.endswith("SHIP-REAL-SOFTWARE.md") for p in excludes), ( + "CLAUDE.md @-imports it, and whether excluding a parent drops its imports is " + "undocumented -- so it is listed explicitly rather than assumed" + ) + + +def test_memory_isolation_does_not_disturb_the_conductor_state(tmp_path: Path): + """Isolation is additive: it must not perturb what the scorers actually read.""" + project = materialize("single-active-epic", tmp_path) + state = json.loads((project / ".conductor" / "state.json").read_text()) + assert state["active"] == "canary-active" + assert (project / "CLAUDE.md").exists(), "pm's own rules block is still written" + + +def test_the_instructions_recorder_actually_works(tmp_path: Path): + """Pipe a real payload through the recorder rather than trusting it. + + The recorder swallows all errors on purpose -- an observability hook must never break the + run it observes. But that swallow also HID a real bug: the first version was `.mjs` and + used `require`/`__dirname`, neither of which exists in an ES module. It threw + ReferenceError, the catch ate it, the log stayed empty, and `no_operator_memory_loaded` + would have passed vacuously forever. This test is why that cannot recur. + """ + import subprocess as sp + + project = materialize("single-active-epic", tmp_path) + recorder = project / ".claude" / "record-instructions.cjs" + assert recorder.exists(), "the InstructionsLoaded recorder must be written into the fixture" + + payload = json.dumps({"memory_type": "User", "file_path": "/home/someone/.claude/CLAUDE.md"}) + sp.run(["node", str(recorder)], input=payload, text=True, check=True, timeout=30) + + log = project / ".claude" / "instructions-loaded.log" + assert log.exists() and log.read_text().strip(), ( + "the recorder produced nothing -- it is broken, and the scorer built on it is vacuous" + ) + assert log.read_text().startswith("User\t/home/someone/.claude/CLAUDE.md") + + +def test_observe_surfaces_a_user_scope_load_so_the_scorer_can_fail(tmp_path: Path): + """The scorer must be able to FAIL. Simulate isolation breaking and confirm it is seen.""" + from observe import observe + + project = materialize("single-active-epic", tmp_path) + assert observe(project)["user_memory_files_loaded"] == [], "clean fixture: nothing user-scope" + + (project / ".claude" / "instructions-loaded.log").write_text( + "Project\t/tmp/p/CLAUDE.md\nUser\t/home/someone/.claude/CLAUDE.md\n" + ) + leaked = observe(project)["user_memory_files_loaded"] + assert leaked == ["/home/someone/.claude/CLAUDE.md"], ( + "a User-scope load must surface, or no_operator_memory_loaded can never fail" + )