feat: gate promotions on predeclared causal evaluation plans

.agents/pm/history/pm-rl-lintruntime.jsonl

@@ -1,2 +1,3 @@
 {"hash_algorithm":"sha256","ts":"2026-10-21T17:42:54.342Z","author":"codex-agent","author_source":"asserted","agent_harness":"codex","agent_model":"gpt-6.1-sol","agent_model_source":"probe","agent_instance":"6ab7e4474260a2bf1a99c414","agent_provenance":{"model":{"value":"gpt-6.1-sol","source":"probe"},"effort":{"value":"high","source":"probe"},"role":{"value":"implementer","source":"argv"},"topic":null},"op":"create","patch":[{"op":"add","path":"/metadata/id","value":"pm-rl-lintruntime"},{"op":"add","path":"/metadata/title","value":"Keep retained runtime artifacts outside authored-source lint"},{"op":"add","path":"/metadata/description","value":"Local npm run lint traverses ignored .agents/pm/runtime execution snapshots and rejects generated packaged declarations there for inline type imports. scripts/eslint.config.ts ignores only root dist, coverage and node_modules. Identical authored sources pass lint in a clean worktree. Preserve retained raw-run evidence and keep the authored source scope and all release thresholds unchanged; define a precise runtime-artifact ignore boundary with regression coverage."},{"op":"add","path":"/metadata/type","value":"Task"},{"op":"add","path":"/metadata/status","value":"open"},{"op":"add","path":"/metadata/priority","value":3},{"op":"add","path":"/metadata/tags","value":[]},{"op":"add","path":"/metadata/created_at","value":"2026-10-21T17:42:54.342Z"},{"op":"add","path":"/metadata/updated_at","value":"2026-10-21T17:42:54.342Z"},{"op":"add","path":"/metadata/author","value":"codex-agent"},{"op":"add","path":"/metadata/acceptance_criteria","value":"A retained packaged declaration under ignored .agents/pm/runtime does not alter authored-source lint results; authored sources and tests remain linted at unchanged strictness; raw execution evidence is preserved."},{"op":"add","path":"/metadata/files","value":[{"path":"scripts/eslint.config.ts","scope":"project"},{"path":".gitignore","scope":"project"}]}],"before_hash":"3cc22dff72be7b14824654a7a64ea62b04799939b2fee54c1b5f52ca60bf6df0","after_hash":"24dd6ad579be0c516edcb7c08409d1c3f9c42fd602d998857ec3ac2247d1025a","item_hash_version":3,"message":"","context":{"agent_provenance_outcomes":{"model":{"status":"resolved","resolver":"codex_session_file","rule_version":"v1"},"effort":{"status":"resolved","resolver":"codex_session_file","rule_version":"v1"},"version":{"status":"unavailable","reason":"harness_unavailable","resolver":"ai_agent_version","rule_version":"v1"}}},"event_class":"substantive","record_hash_version":1,"record_hash":"ce8d21e894570fb05654cb112a7a09bda7fb6f5cdfe2aa2ebe967f17b39ce1c0"}
 {"hash_algorithm":"sha256","ts":"2026-10-21T18:03:11.793Z","author":"codex-agent","author_source":"asserted","agent_harness":"codex","agent_model":"gpt-6.1-sol","agent_model_source":"probe","agent_instance":"6ab7e4474260a2bf1a99c414","agent_provenance":{"model":{"value":"gpt-6.1-sol","source":"probe"},"effort":{"value":"high","source":"probe"},"role":null,"topic":null},"op":"comment_add","patch":[{"op":"replace","path":"/metadata/updated_at","value":"2026-10-21T18:03:11.793Z"},{"op":"add","path":"/metadata/comments","value":[{"created_at":"2026-10-21T18:03:11.793Z","author":"codex-agent","text":"Discovered while validating PR #94: https://github.com/unbraind/pm-rl/pull/94 . Retained ignored execution snapshots trigger the strict inline-import rule in generated packaged declarations, while byte-identical authored sources pass lint and all three complete release variants in clean CI. This follow-up remains separate from corpus output safety. Preserve retained evidence and every authored-source rule/coverage threshold."}]}],"before_hash":"24dd6ad579be0c516edcb7c08409d1c3f9c42fd602d998857ec3ac2247d1025a","after_hash":"e1bd3db16b13e128c7827c07c6981956300364ca51ef8fd501a3d156824cdb17","item_hash_version":3,"context":{"agent_provenance_outcomes":{"model":{"status":"resolved","resolver":"codex_session_file","rule_version":"v1"},"effort":{"status":"resolved","resolver":"codex_session_file","rule_version":"v1"},"version":{"status":"unavailable","reason":"harness_unavailable","resolver":"ai_agent_version","rule_version":"v1"}}},"event_class":"substantive","record_hash_version":1,"record_hash":"673fa5556f7989e0b01bd98b37eb5e666f402b7dd4aeef062e6dcf56a5257d04"}
+{"hash_algorithm":"sha256","ts":"2026-10-22T19:39:06.788Z","author":"codex-agent","author_source":"asserted","agent_harness":"codex","agent_model":"gpt-6.1-sol","agent_model_source":"probe","agent_instance":"c48596e9994322b2614d5e7d","agent_provenance":{"model":{"value":"gpt-6.1-sol","source":"probe"},"effort":{"value":"high","source":"probe"},"role":null,"topic":null},"op":"comment_add","patch":[{"op":"add","path":"/metadata/comments/1","value":{"created_at":"2026-10-22T19:39:06.787Z","author":"codex-agent","text":"Reproduced during PR #97 sealed-evidence review: unmodified npm run check linted generated index.d.ts beneath a retained ignored .agents/pm/runtime/live-meta acceptance output and reported two compiler-generated inline type imports. This is the existing runtime-artifact lint-scope issue, not authored source. Preserved the task-owned synthetic acceptance output outside the package worktree and reran unchanged checks. No lint policy, threshold, fixture data or source was deleted or excluded in the review fix; this follow-up remains open."}},{"op":"replace","path":"/metadata/updated_at","value":"2026-10-22T19:39:06.788Z"}],"before_hash":"e1bd3db16b13e128c7827c07c6981956300364ca51ef8fd501a3d156824cdb17","after_hash":"753fbf565302727a7d39093b8c5d20e8eaac4369ada86483168c1ff99da3690b","item_hash_version":3,"context":{"agent_provenance_outcomes":{"model":{"status":"resolved","resolver":"codex_session_file","rule_version":"v1"},"effort":{"status":"resolved","resolver":"codex_session_file","rule_version":"v1"},"version":{"status":"unavailable","reason":"harness_unavailable","resolver":"ai_agent_version","rule_version":"v1"}}},"event_class":"substantive","record_hash_version":1,"record_hash":"4b82107ec69283c4116fe4ec4bec1007563d462c91b7ca669cbf4f2f1cf290df"}

.agents/pm/tasks/pm-rl-lintruntime.toon

@@ -6,11 +6,12 @@ status: open
 priority: 3
 tags: []
 created_at: "2026-10-21T17:42:54.342Z"
-updated_at: "2026-10-21T18:03:11.793Z"
+updated_at: "2026-10-22T19:39:06.788Z"
 author: codex-agent
 acceptance_criteria: A retained packaged declaration under ignored .agents/pm/runtime does not alter authored-source lint results; authored sources and tests remain linted at unchanged strictness; raw execution evidence is preserved.
-comments[1]{created_at,author,text}:
+comments[2]{created_at,author,text}:
   "2026-10-21T18:03:11.793Z",codex-agent,"Discovered while validating PR #94: https://github.com/unbraind/pm-rl/pull/94 . Retained ignored execution snapshots trigger the strict inline-import rule in generated packaged declarations, while byte-identical authored sources pass lint and all three complete release variants in clean CI. This follow-up remains separate from corpus output safety. Preserve retained evidence and every authored-source rule/coverage threshold."
+  "2026-10-22T19:39:06.787Z",codex-agent,"Reproduced during PR #97 sealed-evidence review: unmodified npm run check linted generated index.d.ts beneath a retained ignored .agents/pm/runtime/live-meta acceptance output and reported two compiler-generated inline type imports. This is the existing runtime-artifact lint-scope issue, not authored source. Preserved the task-owned synthetic acceptance output outside the package worktree and reran unchanged checks. No lint policy, threshold, fixture data or source was deleted or excluded in the review fix; this follow-up remains open."
 files[2]{path,scope}:
   scripts/eslint.config.ts,project
   .gitignore,project

README.md

@@ -579,3 +579,102 @@ It does not mutate a tracker, grant a budget, execute an LLM, or launch a hosted
 job. Evaluation is an adaptive validation set; a separate final benchmark is
 needed after repeated selection. See [the execution contract](RECURSIVE_TRAINING.md)
 for the local process and Compose adapters; [the decision adapter](docs/durable-systemone-validation.md) documents the implemented frozen-model calibration trainer.
+
+## Predeclared causal evaluation plans
+
+Declare the target population, selection rule, member-to-correlation-unit mapping,
+exact observation and independent-unit counts, objective, bounded reward scale,
+criterion and fixed stopping rule **before creating a candidate or its collection
+runs**. The plan also pins the incumbent generation's provenance hash. The
+[example plan](examples/evaluation-plan.json) and [paired observations](examples/evaluation-observations.json)
+form a powered synthetic positive control:
+
+```bash
+pm init rl --defaults
+pm rl generation register rl-seed --base-checkpoint checkpoint
+pm rl eval plan register --file examples/evaluation-plan.json
+pm rl eval plan show rl-candidate --json
+# Create the collection run and register rl-candidate with parent rl-seed.
+pm rl eval plan evaluate rl-candidate --file examples/evaluation-observations.json --json
+# Supply matching held-out scores to the existing generation promotion command.
+pm rl generation promote rl-candidate --approval rl-approval --scores scores.json --evidence "Predeclared paired comparison"
+```
+
+Use full canonical generation IDs and the incumbent's `spec_hash` from registration
+as `incumbent_version`. Copy and adjust the example for your tracker before declaring
+it. TOON is the default; `--json` returns the same compact summary: comparison hash,
+observation/unit counts, equal-unit means, statistical verdict and remedy. Cohorts
+and raw observations remain in the Plan and result Decision, available through
+complete item reads (`pm get <plan-id> --depth standard --output-limit unbounded
+--output-budget unbounded`). Output size does not grow with cohort size. For promotion,
+set the held-out score's `seed_set` to the registered comparison hash, `scale`
+to the declared reward-bound width, and `value` to exactly equal the report's
+`candidate_mean`. Copy that mean unrounded from the `--json` output, since TOON
+may format numbers differently. All three fields are checked against the frozen
+plan and sealed evidence; rounding the mean causes an `eval_plan_evidence` refusal.
+
+Each member supplies one candidate and incumbent outcome. The evaluator first
+averages within each declared correlation unit, then weights units equally and
+uses the **number of units** in the existing Hoeffding promotion gate. A hundred
+related items in one project cannot count as a hundred independent samples.
+Rewards are centered on the declared lower bound and rescaled to `[0, 1]` before
+averaging and computing confidence bounds; the effect threshold is divided by
+the declared reward width. This keeps large common offsets and subnormal scales
+from erasing uncertainty. Ties never promote. Reported `candidate_mean` and
+`incumbent_mean` retain the declared reward units; the statistical `verdict`
+bounds, epsilons, margin and reason use the normalized units.
+Unknown members, duplicates, supplied aggregate counts, changed objective/scale,
+missing observations, extra observations and adaptive stopping are refused. Up to
+10,000 members are supported. This fixed-stop format allows one sealed paired
+look per plan, including a statistical refusal; retrying identical evidence is
+idempotent. Changing a population, grouping, stopping size, criterion or incumbent
+provenance changes comparison identity and requires a fresh plan/candidate.
+Plan reads, evaluation retries and promotions compare sealed fields against the
+SDK's verified version-one history snapshot. Replacing observations or a plan
+and recomputing its hash is refused with `eval_plan_drift`. Missing, corrupt or
+compacted-away original history also refuses; unrelated annotations remain allowed.
+Candidate provenance is checked against its creation snapshot, while creation
+order uses the original candidate and collection-run timestamps. Deleting a plan
+and removing the candidate's pin cannot discard a reservation retained in history.
+This relies on the SDK's append-only history: privileged rewriting of the original
+history itself is outside this enforcement boundary.
+
+`rl generation register` pins an existing plan in generation provenance. Both
+manual and controller promotions re-read the plan and sealed outcomes inside the
+existing promotion transaction; missing plans, comparison drift, changed parents,
+late predeclarations, mismatched held-out scores and refused bounds stop promotion
+with stable `eval_plan_*` codes and a remedy. The original contamination and
+approval-budget checks still apply. Existing unplanned workflows retain their
+legacy behavior and carry no predeclared causal-evaluation claim. A reserved
+candidate remains gated even if its caller omits plan flags; a generation pin
+also makes a deleted plan a refusal.
+
+The typed SDK exports `EvaluationPlan`, `EvaluationObservation`,
+`EvaluationPlanReport`, `EvaluationPlanRefusal`, `parseEvaluationPlan`,
+`evaluationComparison`, `registerEvaluationPlan`, `readEvaluationPlan`,
+`evaluatePlan`, `recordPlanEvaluation` and `enforceEvaluationPlan`. Tracker methods
+accept a real `PmClient`; `evaluatePlan` is pure. Plans and results use core Plan
+and Decision records; this does not replace pm-jev's decision-model calibration
+or the separate powered-corpus dataset work.
+
+Independence between declared units, truthful cohort membership, outcome
+measurement and appropriate target-population selection remain experiment-design
+assumptions. Paired policy replay alone does not establish a randomized causal
+effect in a deployed population. The gate enforces the declared design and a
+conservative bound; it cannot detect semantic correlations hidden from the plan
+or prevent external peeking. Repeated experiments require their own selection or
+multiplicity analysis. [Public tracker validation](docs/evaluation-plan-evidence.json)
+records aggregate replay measurements and the unchanged read-only source digest.
+The acceptance runner takes a read-only public tracker copy, an output report,
+the public source commit and a packed package archive:
+`node test/evaluation-plan.acceptance.ts <tracker-copy> <report.json> <commit> <package.tgz>`.
+It installs the archive into a plain scratch tracker and runs the real JSON/TOON
+CLI with Git absent from `PATH`.
+
+## Without git
+
+Tracker-based environments, runs, evaluations, predeclared plans, generation
+promotion and lineage work in plain directories without Git. Public-history
+dataset/improver builders and Git-tree episode replay need Git provenance. Plan
+registration, evaluation and promotion are tested with Git absent from `PATH` and
+an explicit discovery ceiling.

docs/evaluation-plan-evidence.json

@@ -0,0 +1,25 @@
+{
+  "source": "https://github.com/unbraind/pm-rl",
+  "commit": "ce948522e18a37f92e9d3b345371ac924c5fa305",
+  "snapshot": "d3f1492a7acb6e9d86ae9c62045349e2f627ed29383bb0e502d350087a4e2e96",
+  "source_digest": "ae2abd5b4ee61a7e3a8f03066429a95b031883aba761fafc9284565ba5e90ed2",
+  "source_unchanged": true,
+  "source_read_only": true,
+  "git_repository_required": false,
+  "cli_json_bytes": 780,
+  "cli_toon_bytes": 280,
+  "plan": "rl-candidate-evaluation-plan",
+  "comparison": "c9b4bdee1a5410c663d5495f4015a834676d0fb7b804904c6eed3d5b8bbcdb09",
+  "observations": 95,
+  "independent_units": 1,
+  "candidate_mean": 0.7684210526315792,
+  "incumbent_mean": 0.2315789473684211,
+  "verdict": {
+    "decision": "refuse",
+    "code": "insufficient_samples",
+    "reason": "Promotion refused: the candidate evidence carries 1 sample(s), below the required minimum of 10. A noisy sample below the minimum count is not statistically bounded improvement."
+  },
+  "remedy": "Register a new plan and fresh candidate before collecting a different cohort or making another evaluation look.",
+  "naive_episode_gate": "promote",
+  "policy": "Constant closed versus constant not-closed classification; descriptive replay only. One repository is one correlation unit, with no causal generalization or trainer improvement claim."
+}