From 428ffa7051572a626b55b4cd2023856ea44a1cc0 Mon Sep 17 00:00:00 2001 From: Tyler Kendrick <145080887+Tyler-R-Kendrick@users.noreply.github.com> Date: Sun, 26 Jul 2026 23:40:20 -0500 Subject: [PATCH] SLM-431: land SLM-317 repair-hybrid harness + powered Wilson rerun (LAR0-06) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Land the SLM-317 do-no-harm AR->repair hybrid harness byte-identical from unmerged branch commit 48e5cadc (harness module, runner, tests, frozen 20260724 docs); 20 original tests pass unmodified against main HEAD. Add the additive, default-off --min-pass-rate Wilson power-rule CLI mirroring SLM-421's run_slm138 power_rule (per-seed pass/fail -> wilson_interval -> repair_positive / repair_negative / inconclusive_underpowered), with 7 new deterministic helper tests. power_rule=None preserves the original 2-seed behavior byte-for-byte. STOP RULE: the powered rerun cannot execute on main — the runner's historical/improved arms need TreeEditDiffusionConfig value_label_mode / stop_slot_accounting knobs that exist only in the branch's forked tree_edit_diffusion.py (unmerged SLM-305/308/310). Reported honestly per the issue's falsification rule; no rerun artifact, no disposition, LAR3's second reopening condition remains unmet. Brain note updated. --- .../repo/recursive-recurrence-health.md | 37 +- .../iter-slm317-repair-hybrid-20260724.json | 899 ++++++++++++++++++ .../iter-slm317-repair-hybrid-20260724.md | 53 ++ scripts/run_slm317_repair_hybrid.py | 857 +++++++++++++++++ .../experiments/slm317_repair_hybrid.py | 446 +++++++++ src/slm_training/resources/versions.json | 30 +- .../experiments/test_slm317_repair_hybrid.py | 387 ++++++++ 7 files changed, 2701 insertions(+), 8 deletions(-) create mode 100644 docs/design/iter-slm317-repair-hybrid-20260724.json create mode 100644 docs/design/iter-slm317-repair-hybrid-20260724.md create mode 100644 scripts/run_slm317_repair_hybrid.py create mode 100644 src/slm_training/harnesses/experiments/slm317_repair_hybrid.py create mode 100644 tests/test_harnesses/experiments/test_slm317_repair_hybrid.py diff --git a/docs/brains/repo/recursive-recurrence-health.md b/docs/brains/repo/recursive-recurrence-health.md index 75b1f69db..508acabff 100644 --- a/docs/brains/repo/recursive-recurrence-health.md +++ b/docs/brains/repo/recursive-recurrence-health.md @@ -3,8 +3,8 @@ type: concept status: active tags: [recurrence, diagnostics, fixture] created: 2026-07-23 -updated: 2026-07-25 -linear: SLM-282, SLM-421 +updated: 2026-07-27 +linear: SLM-282, SLM-421, SLM-317, SLM-431 design: docs/design/iter-slm282-recurrence-health-20260723.md, docs/design/iter-slm282-recurrence-health-powered-rerun-20260725.md sources: "[[deeploop-source]], [[training-free-looped-transformers-source]], https://arxiv.org/abs/2106.14342" --- @@ -56,8 +56,31 @@ systematic property of the core. **Net effect on LAR3**: only one of the two PR #853-#856 reopening conditions is now met. LAR3 stays closed — the second condition (a passing SLM-317-style -valid-state repair advancement screen) is untouched by this issue; SLM-317's -own harness is not yet merged to `main`, so its powered rerun is a separate, -not-yet-landable follow-up. No ship, checkpoint, or production-default claim -follows from either record; residual_delta remains a fixture-only -counterfactual, never a production default. +valid-state repair advancement screen) is unresolved; see the SLM-431 blocker +below. No ship, checkpoint, or production-default claim follows from either +record; residual_delta remains a fixture-only counterfactual, never a +production default. + +**SLM-431 (2026-07-27): harness landed; powered rerun BLOCKED by unmerged +branch-only model dependencies (stop rule).** The SLM-317 harness +(`src/slm_training/harnesses/experiments/slm317_repair_hybrid.py`), +its runner (`scripts/run_slm317_repair_hybrid.py`), its tests, and the frozen +`docs/design/iter-slm317-repair-hybrid-20260724.{md,json}` record were landed +byte-identical from unmerged branch commit `48e5cadc`; the 20 original tests +pass unmodified against main HEAD, and SLM-431 added the same additive, +default-off `--min-pass-rate` Wilson power-rule surface SLM-421 added to +`run_slm138_recursive_denoiser_fixture.py` (7 new deterministic helper +tests). However the powered rerun itself **cannot execute on main**: the +runner's improved/historical repair arms construct `TreeEditDiffusionConfig` +with `value_label_mode` and `stop_slot_accounting` knobs that exist **only** +in the branch's forked `tree_edit_diffusion.py` (unmerged SLM-305/308/310 +branch commits; main's model has neither the knobs nor the concepts under +any name). Running the rerun would require porting those branch model +changes — a behavior change the issue explicitly forbids. Per the issue's +falsification/stop rule this is reported honestly and the rerun was **not** +forced: no `iter-slm317-repair-hybrid-powered-rerun-*` artifact exists, no +disposition (repair_positive / repair_negative / inconclusive_underpowered) +was produced, and the second LAR3-reopening condition remains **unmet**. Next +step: a decision issue on whether to land/port the SLM-305/308/310 model +work (or re-express the historical/improved arms against main's model) before +any powered rerun is attempted. diff --git a/docs/design/iter-slm317-repair-hybrid-20260724.json b/docs/design/iter-slm317-repair-hybrid-20260724.json new file mode 100644 index 000000000..eddd68faf --- /dev/null +++ b/docs/design/iter-slm317-repair-hybrid-20260724.json @@ -0,0 +1,899 @@ +{ + "experiment": "slm317-repair-hybrid", + "issue": "SLM-317", + "question": "Does a do-no-harm AR\u2192repair hybrid (deterministic commit rule) improve paired hard-valid outcomes over AR only without ever damaging a valid AR program?", + "preregistered": { + "min_useful_effect": 0.05, + "safety_rule": "invalid-over-valid count of ar_repair_improved vs ar_only on the frozen safety set (all eval examples) must be exactly 0", + "value_rule": "paired hard-valid improvement of ar_repair_improved over ar_only: Wilson 95% lower bound of improvements/n_paired must exceed 0.05 with zero damages", + "disposition_rule": "repair_positive iff safety AND value AND reachability gates pass; repair_negative iff safety fails OR the value Wilson UPPER bound is below the minimum useful effect (effect ruled out); else inconclusive", + "historical_deviation": "pre-SLM-305 4-action edit space is not reproducible on this branch (edit space extended in place, no legacy-subset knob); the historical arm keeps mutation_count value labels + legacy STOP-slot accounting only" + }, + "arms": [ + "ar_only", + "repair_only", + "ar_repair_historical", + "ar_repair_improved", + "oracle_commit" + ], + "promotable_arms": [ + "ar_only", + "ar_repair_historical", + "ar_repair_improved", + "repair_only" + ], + "config": { + "n_train_decisions": 16, + "n_eval_decisions": 8, + "seeds": [ + 0, + 1 + ], + "steps": 6, + "lr": 0.003, + "scorer_steps": 20, + "fixture_config_overrides": { + "d_model": 64, + "denoiser_layers": 2, + "context_layers": 1, + "max_chain": 3, + "max_search_steps": 6, + "beam_width": 3, + "expand_per_state": 3 + }, + "torch_num_threads": 1 + }, + "arm_summaries": { + "ar_only": { + "n": 16, + "valid_rate": 1.0, + "mean_hard_rank": 3.0 + }, + "repair_only": { + "n": 16, + "valid_rate": 1.0, + "mean_hard_rank": 3.0 + }, + "ar_repair_historical": { + "n": 16, + "valid_rate": 1.0, + "mean_hard_rank": 3.0 + }, + "ar_repair_improved": { + "n": 16, + "valid_rate": 1.0, + "mean_hard_rank": 3.0 + }, + "oracle_commit": { + "n": 16, + "valid_rate": 1.0, + "mean_hard_rank": 3.0 + } + }, + "paired_vs_ar_only": { + "repair_only": { + "both_valid": 16, + "a_valid_b_invalid": 0, + "a_invalid_b_valid": 0, + "both_invalid": 0, + "unpaired": 0 + }, + "ar_repair_historical": { + "both_valid": 16, + "a_valid_b_invalid": 0, + "a_invalid_b_valid": 0, + "both_invalid": 0, + "unpaired": 0 + }, + "ar_repair_improved": { + "both_valid": 16, + "a_valid_b_invalid": 0, + "a_invalid_b_valid": 0, + "both_invalid": 0, + "unpaired": 0 + }, + "oracle_commit": { + "both_valid": 16, + "a_valid_b_invalid": 0, + "a_invalid_b_valid": 0, + "both_invalid": 0, + "unpaired": 0 + } + }, + "invalid_over_valid_per_arm": { + "repair_only": 0, + "ar_repair_historical": 0, + "ar_repair_improved": 0, + "oracle_commit": 0 + }, + "commit_reason_counts": { + "ar_repair_historical": { + "hard_regression": 16 + }, + "ar_repair_improved": { + "no_improvement": 16 + }, + "oracle_commit": { + "oracle_no_gain": 16 + } + }, + "gates": { + "safety": { + "rule": "invalid-over-valid count of ar_repair_improved vs ar_only on the frozen safety set (all eval examples) must be exactly 0", + "invalid_over_valid": 0, + "pass": true + }, + "value": { + "rule": "paired hard-valid improvement of ar_repair_improved over ar_only: Wilson 95% lower bound of improvements/n_paired must exceed 0.05 with zero damages", + "improvements": 0, + "n_paired": 16, + "wilson": { + "n": 16, + "estimate": 0.0, + "low": 0.0, + "high": 0.19360768053443644, + "confidence_level": 0.95 + }, + "min_useful_effect": 0.05, + "pass": false + }, + "reachability": { + "artifacts": { + "docs/design/iter-slm299-edit-reachability-20260724.json": true, + "docs/design/iter-slm299-edit-reachability-20260724.md": true, + "docs/design/iter-slm291-evidence-bundles-20260724.json": true, + "docs/design/iter-slm291-evidence-bundles-20260724.md": true + }, + "pass": true + } + }, + "wilson_damage": { + "n": 16, + "estimate": 0.0, + "low": 0.0, + "high": 0.19360768053443644, + "confidence_level": 0.95 + }, + "oracle_upper_bound": { + "valid_rate": 1.0, + "sanity": true + }, + "per_example_outcomes": [ + { + "record_id": "decision-0", + "arm_id": "ar_only", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-0", + "arm_id": "repair_only", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-0", + "arm_id": "ar_repair_historical", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "hard_regression" + }, + { + "record_id": "decision-0", + "arm_id": "ar_repair_improved", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "no_improvement" + }, + { + "record_id": "decision-0", + "arm_id": "oracle_commit", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "oracle_no_gain" + }, + { + "record_id": "decision-1", + "arm_id": "ar_only", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-1", + "arm_id": "repair_only", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-1", + "arm_id": "ar_repair_historical", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "hard_regression" + }, + { + "record_id": "decision-1", + "arm_id": "ar_repair_improved", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "no_improvement" + }, + { + "record_id": "decision-1", + "arm_id": "oracle_commit", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "oracle_no_gain" + }, + { + "record_id": "decision-2", + "arm_id": "ar_only", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":card\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-2", + "arm_id": "repair_only", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":card\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-2", + "arm_id": "ar_repair_historical", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":card\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "hard_regression" + }, + { + "record_id": "decision-2", + "arm_id": "ar_repair_improved", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":card\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "no_improvement" + }, + { + "record_id": "decision-2", + "arm_id": "oracle_commit", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":card\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "oracle_no_gain" + }, + { + "record_id": "decision-3", + "arm_id": "ar_only", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-3", + "arm_id": "repair_only", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-3", + "arm_id": "ar_repair_historical", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "hard_regression" + }, + { + "record_id": "decision-3", + "arm_id": "ar_repair_improved", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "no_improvement" + }, + { + "record_id": "decision-3", + "arm_id": "oracle_commit", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "oracle_no_gain" + }, + { + "record_id": "decision-4", + "arm_id": "ar_only", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-4", + "arm_id": "repair_only", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-4", + "arm_id": "ar_repair_historical", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "hard_regression" + }, + { + "record_id": "decision-4", + "arm_id": "ar_repair_improved", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "no_improvement" + }, + { + "record_id": "decision-4", + "arm_id": "oracle_commit", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "oracle_no_gain" + }, + { + "record_id": "decision-5", + "arm_id": "ar_only", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-5", + "arm_id": "repair_only", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-5", + "arm_id": "ar_repair_historical", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "hard_regression" + }, + { + "record_id": "decision-5", + "arm_id": "ar_repair_improved", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "no_improvement" + }, + { + "record_id": "decision-5", + "arm_id": "oracle_commit", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "oracle_no_gain" + }, + { + "record_id": "decision-6", + "arm_id": "ar_only", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-6", + "arm_id": "repair_only", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-6", + "arm_id": "ar_repair_historical", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "hard_regression" + }, + { + "record_id": "decision-6", + "arm_id": "ar_repair_improved", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "no_improvement" + }, + { + "record_id": "decision-6", + "arm_id": "oracle_commit", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "oracle_no_gain" + }, + { + "record_id": "decision-7", + "arm_id": "ar_only", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-7", + "arm_id": "repair_only", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-7", + "arm_id": "ar_repair_historical", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "hard_regression" + }, + { + "record_id": "decision-7", + "arm_id": "ar_repair_improved", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "no_improvement" + }, + { + "record_id": "decision-7", + "arm_id": "oracle_commit", + "seed": 0, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "oracle_no_gain" + }, + { + "record_id": "decision-0", + "arm_id": "ar_only", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-0", + "arm_id": "repair_only", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-0", + "arm_id": "ar_repair_historical", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "hard_regression" + }, + { + "record_id": "decision-0", + "arm_id": "ar_repair_improved", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "no_improvement" + }, + { + "record_id": "decision-0", + "arm_id": "oracle_commit", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "oracle_no_gain" + }, + { + "record_id": "decision-1", + "arm_id": "ar_only", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-1", + "arm_id": "repair_only", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-1", + "arm_id": "ar_repair_historical", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "hard_regression" + }, + { + "record_id": "decision-1", + "arm_id": "ar_repair_improved", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "no_improvement" + }, + { + "record_id": "decision-1", + "arm_id": "oracle_commit", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "oracle_no_gain" + }, + { + "record_id": "decision-2", + "arm_id": "ar_only", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":card\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-2", + "arm_id": "repair_only", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":card\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-2", + "arm_id": "ar_repair_historical", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":card\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "hard_regression" + }, + { + "record_id": "decision-2", + "arm_id": "ar_repair_improved", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":card\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "no_improvement" + }, + { + "record_id": "decision-2", + "arm_id": "oracle_commit", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":card\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "oracle_no_gain" + }, + { + "record_id": "decision-3", + "arm_id": "ar_only", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-3", + "arm_id": "repair_only", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-3", + "arm_id": "ar_repair_historical", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "hard_regression" + }, + { + "record_id": "decision-3", + "arm_id": "ar_repair_improved", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "no_improvement" + }, + { + "record_id": "decision-3", + "arm_id": "oracle_commit", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "oracle_no_gain" + }, + { + "record_id": "decision-4", + "arm_id": "ar_only", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-4", + "arm_id": "repair_only", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-4", + "arm_id": "ar_repair_historical", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "hard_regression" + }, + { + "record_id": "decision-4", + "arm_id": "ar_repair_improved", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "no_improvement" + }, + { + "record_id": "decision-4", + "arm_id": "oracle_commit", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "oracle_no_gain" + }, + { + "record_id": "decision-5", + "arm_id": "ar_only", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-5", + "arm_id": "repair_only", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-5", + "arm_id": "ar_repair_historical", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "hard_regression" + }, + { + "record_id": "decision-5", + "arm_id": "ar_repair_improved", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "no_improvement" + }, + { + "record_id": "decision-5", + "arm_id": "oracle_commit", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":text\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "oracle_no_gain" + }, + { + "record_id": "decision-6", + "arm_id": "ar_only", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-6", + "arm_id": "repair_only", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-6", + "arm_id": "ar_repair_historical", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "hard_regression" + }, + { + "record_id": "decision-6", + "arm_id": "ar_repair_improved", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "no_improvement" + }, + { + "record_id": "decision-6", + "arm_id": "oracle_commit", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":button\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "oracle_no_gain" + }, + { + "record_id": "decision-7", + "arm_id": "ar_only", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-7", + "arm_id": "repair_only", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": null + }, + { + "record_id": "decision-7", + "arm_id": "ar_repair_historical", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "hard_regression" + }, + { + "record_id": "decision-7", + "arm_id": "ar_repair_improved", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "no_improvement" + }, + { + "record_id": "decision-7", + "arm_id": "oracle_commit", + "seed": 1, + "final_source": "root = Stack([n0], \"column\")\nn0 = TextContent(\":stack\")", + "v1_verdict": true, + "hard_rank": 3, + "commit_reason": "oracle_no_gain" + } + ], + "decisions_path": "outputs/slm317/commit_decisions.jsonl", + "disposition": "inconclusive", + "lar3_statement": "LAR3 (learned repair integration) remains OPEN but NOT advanced: the screen neither cleared the value gate nor ruled the effect out at fixture power; a powered rerun is required before LAR3 can open or close.", + "wall_seconds": 67.08616952600278, + "honesty": "Fixture-scale wiring screen: the frozen corpus is the SLM-155 synthetic decision fixture (n eval decisions below any ship-gate prompt count), models are tiny CPU fixtures trained for a handful of steps, and no ship-gate claim is made. The commit rule is deterministic and metamorphism-invariant by construction; the screen measures the mechanism, not production quality. The historical arm's pre-SLM-305 4-action space is declared non-reproducible on this branch (see preregistered deviation).", + "version_stamp": { + "stamp_schema": "version_stamp/v1", + "code_commit": "f6b173105eb2f3e2e5fa2b5fea7e3af452cc2b83", + "code_dirty": true, + "components": { + "harness.experiments.slm317_repair_hybrid": "v1", + "harness.experiments.slm155_factorization_comparison": "v2", + "harness.experiments.slm299_edit_reachability": "v4" + }, + "stamped_at": "2026-07-25T00:11:13.905812+00:00" + } +} diff --git a/docs/design/iter-slm317-repair-hybrid-20260724.md b/docs/design/iter-slm317-repair-hybrid-20260724.md new file mode 100644 index 000000000..e7e2d5e05 --- /dev/null +++ b/docs/design/iter-slm317-repair-hybrid-20260724.md @@ -0,0 +1,53 @@ +# SLM-317 (LAR2-06): do-no-harm AR→repair hybrid screen + +**Disposition: `inconclusive`** — LAR3 (learned repair integration) remains OPEN but NOT advanced: the screen neither cleared the value gate nor ruled the effect out at fixture power; a powered rerun is required before LAR3 can open or close. + +Fixture-scale mechanism screen; **not a ship claim**. + +## Preregistered (locked before results) + +- safety: invalid-over-valid count of ar_repair_improved vs ar_only on the frozen safety set (all eval examples) must be exactly 0 +- value: paired hard-valid improvement of ar_repair_improved over ar_only: Wilson 95% lower bound of improvements/n_paired must exceed 0.05 with zero damages +- disposition: repair_positive iff safety AND value AND reachability gates pass; repair_negative iff safety fails OR the value Wilson UPPER bound is below the minimum useful effect (effect ruled out); else inconclusive +- declared deviation: pre-SLM-305 4-action edit space is not reproducible on this branch (edit space extended in place, no legacy-subset knob); the historical arm keeps mutation_count value labels + legacy STOP-slot accounting only + +## Arms (matched examples + seeds + budgets) + +| arm | n | hard-valid rate | mean hard rank | invalid-over-valid vs AR | +| --- | --- | --- | --- | --- | +| ar_only | 16 | 1.000 | 3.000 | — | +| repair_only | 16 | 1.000 | 3.000 | 0 | +| ar_repair_historical | 16 | 1.000 | 3.000 | 0 | +| ar_repair_improved | 16 | 1.000 | 3.000 | 0 | +| oracle_commit | 16 | 1.000 | 3.000 | 0 | + +## Paired outcomes vs `ar_only` (per-example, nothing aggregated away) + +| arm | both valid | AR valid, arm invalid | AR invalid, arm valid | both invalid | unpaired | +| --- | --- | --- | --- | --- | --- | +| repair_only | 16 | 0 | 0 | 0 | 0 | +| ar_repair_historical | 16 | 0 | 0 | 0 | 0 | +| ar_repair_improved | 16 | 0 | 0 | 0 | 0 | +| oracle_commit | 16 | 0 | 0 | 0 | 0 | + +## Advancement gates + +- **Safety**: invalid-over-valid = 0 (must be 0) → PASS +- **Value**: improvements 0/16, Wilson 95% [0.0, 0.19360768053443644] vs minimum useful effect 0.05 → FAIL +- **Reachability/provenance**: slm299/291 artifacts present → PASS + +## Commit reasons + +| arm | reason | count | +| --- | --- | --- | +| ar_repair_historical | hard_regression | 16 | +| ar_repair_improved | no_improvement | 16 | +| oracle_commit | oracle_no_gain | 16 | + +Oracle commit upper bound hard-valid rate: 1.000 (sanity ≥ improved hybrid: True). + +Durable per-example commit decisions: `outputs/slm317/commit_decisions.jsonl`. + +## Honesty + +Fixture-scale wiring screen: the frozen corpus is the SLM-155 synthetic decision fixture (n eval decisions below any ship-gate prompt count), models are tiny CPU fixtures trained for a handful of steps, and no ship-gate claim is made. The commit rule is deterministic and metamorphism-invariant by construction; the screen measures the mechanism, not production quality. The historical arm's pre-SLM-305 4-action space is declared non-reproducible on this branch (see preregistered deviation). diff --git a/scripts/run_slm317_repair_hybrid.py b/scripts/run_slm317_repair_hybrid.py new file mode 100644 index 000000000..b5bd9b0f2 --- /dev/null +++ b/scripts/run_slm317_repair_hybrid.py @@ -0,0 +1,857 @@ +#!/usr/bin/env python3 +"""SLM-317 (LAR2-06): valid-state repair advancement screen (AR→repair hybrid). + +Runs the preregistered do-no-harm screen over matched arms on the frozen +SLM-155 fixture decision corpus (identical examples + seeds + budgets): + +- ``ar_only`` — the SLM-155 AR legal-action path; final = AR program. +- ``repair_only`` — X22 value-guided beam decode from the minimal seed. +- ``ar_repair_historical`` — AR source → repair decode under the pre-LAR2 + config (``mutation_count`` value labels, legacy STOP-slot accounting). The + pre-SLM-305 4-action edit space is NOT reproducible on this branch (the + space was extended in place; no legacy-subset knob exists) — declared + deviation, the arm keeps the historical value/stop config only. +- ``ar_repair_improved`` — AR source → repair decode under this branch's + config (``bounded_distance`` oracle value labels + corrected STOP + accounting), then the do-no-harm commit rule with the policy value head as + the calibrated learned score. +- ``oracle_commit`` — commits exactly when the candidate's hard evidence is + strictly better (non-promotable upper bound). + +Every final is scored on the LAR1-02 unified ladder (official parse → output +contract → meaningful_program_v1). Paired per-example outcomes key on +(record_id, seed); invalid-over-valid counts are reported per arm and never +aggregated away. Advancement gates: Safety = 0 invalid-over-valid on the +frozen safety set; Value = paired improvement over AR with Wilson lower bound +above the preregistered minimum useful effect (0.05); Reachability/provenance += LAR2 artifacts present (slm299 reachability, slm291 evidence bundles). +Exactly one disposition: repair_positive | repair_negative | inconclusive, +with an explicit LAR3 open/close statement. + +Writes ``docs/design/iter-slm317-repair-hybrid-20260724.{json,md}`` and +durable commit decisions under ``outputs/slm317/``. + +Example: + python -m scripts.run_slm317_repair_hybrid --steps 8 +""" + +from __future__ import annotations + +import argparse +import json +import time +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from slm_training.evals.power_protocol import wilson_interval +from slm_training.harnesses.experiments.slm155_factorization_comparison import ( + _program_from_actions, + paired_validity_table, +) +from slm_training.harnesses.experiments.slm317_repair_hybrid import ( + DISPOSITION_INCONCLUSIVE, + DISPOSITION_NEGATIVE, + DISPOSITION_POSITIVE, + EXPERIMENT_ID, + CommitDecision, + commit, + hard_evidence, + oracle_commit, + repair_decode, + state_value, +) +from slm_training.models.legal_action_scorer import ( + LegalActionScorerConfig, + make_fixture_decisions, + train_fixture_scorer, +) +from slm_training.versioning import build_version_stamp + +DEFAULT_JSON_OUT = Path("docs/design/iter-slm317-repair-hybrid-20260724.json") +DEFAULT_MD_OUT = Path("docs/design/iter-slm317-repair-hybrid-20260724.md") +DEFAULT_DECISIONS_OUT = Path("outputs/slm317/commit_decisions.jsonl") + +ARMS = ( + "ar_only", + "repair_only", + "ar_repair_historical", + "ar_repair_improved", + "oracle_commit", +) +PROMOTABLE_ARMS = {"ar_only", "repair_only", "ar_repair_historical", "ar_repair_improved"} + +# LAR2 reachability/provenance artifacts this screen builds on. +REACHABILITY_ARTIFACTS = ( + "docs/design/iter-slm299-edit-reachability-20260724.json", + "docs/design/iter-slm299-edit-reachability-20260724.md", + "docs/design/iter-slm291-evidence-bundles-20260724.json", + "docs/design/iter-slm291-evidence-bundles-20260724.md", +) + +# PREREGISTERED (locked before any run; deviations append-only/exploratory). +PREREGISTERED = { + "min_useful_effect": 0.05, + "safety_rule": ( + "invalid-over-valid count of ar_repair_improved vs ar_only on the " + "frozen safety set (all eval examples) must be exactly 0" + ), + "value_rule": ( + "paired hard-valid improvement of ar_repair_improved over ar_only: " + "Wilson 95% lower bound of improvements/n_paired must exceed 0.05 " + "with zero damages" + ), + "disposition_rule": ( + "repair_positive iff safety AND value AND reachability gates pass; " + "repair_negative iff safety fails OR the value Wilson UPPER bound is " + "below the minimum useful effect (effect ruled out); else inconclusive" + ), + "historical_deviation": ( + "pre-SLM-305 4-action edit space is not reproducible on this branch " + "(edit space extended in place, no legacy-subset knob); the " + "historical arm keeps mutation_count value labels + legacy STOP-slot " + "accounting only" + ), +} + +# SLM-431 (LAR0-06): additive, default-off powered-rerun decision surface, +# structurally mirroring SLM-421's power_rule addition to +# scripts/run_slm138_recursive_denoiser_fixture.py. When --min-pass-rate is +# supplied, the value decision moves from the original n=2 per-example Wilson +# screen to a per-seed pass/fail Wilson interval over seeds DISJOINT from the +# original [0, 1]; the corpus, matched arms, do-no-harm commit rule, safety +# gate, and reachability gate stay byte-identical. power_rule=None (the +# default) preserves the original 2-seed behavior byte-for-byte. +DISPOSITION_UNDERPOWERED = "inconclusive_underpowered" + +POWER_PREREGISTERED = { + "seed_pass_rule": ( + "a seed passes iff ar_repair_improved shows >=1 paired improvement " + "over ar_only (AR invalid, arm valid) with 0 damages (AR valid, arm " + "invalid) on that seed's frozen SLM-155 eval decisions" + ), + "disposition_rule": ( + "repair_positive iff safety AND reachability gates pass AND the " + "Wilson 95% lower bound of the per-seed pass rate clears " + "min_pass_rate; repair_negative iff safety fails OR the Wilson UPPER " + "bound falls below min_pass_rate (pass rate ruled out); else " + "inconclusive_underpowered" + ), + "seed_disjointness": ( + "powered-rerun seeds must be disjoint from SLM-317's original " + "[0, 1] (mirrors SLM-421's 2..21)" + ), +} + + +def _today_slug() -> str: + return datetime.now(timezone.utc).strftime("%Y%m%d") + + +def power_disposition( + *, + passed_seeds: int, + n_seeds: int, + min_pass_rate: float, + safety_pass: bool, + reachability_pass: bool, + confidence_level: float = 0.95, +) -> tuple[str, dict[str, Any]]: + """Preregistered Wilson-interval power decision (SLM-431, default-off). + + Locked before any new seed's results are read. Returns the disposition + and the Wilson interval of the per-seed pass rate. Mirrors SLM-421's + rule: positive iff the interval lower bound clears ``min_pass_rate`` + (with safety AND reachability passing), negative iff safety fails or the + interval upper bound falls below it (the pass rate is ruled out), else + ``inconclusive_underpowered``. + """ + min_pass_rate = float(min_pass_rate) + if not 0.0 < min_pass_rate < 1.0: + raise ValueError("power_rule['min_pass_rate'] must be in (0, 1)") + interval = wilson_interval( + passed_seeds, n_seeds, confidence_level=confidence_level + ) + if not safety_pass or ( + interval["high"] is not None and interval["high"] < min_pass_rate + ): + disposition = DISPOSITION_NEGATIVE + elif ( + reachability_pass + and interval["low"] is not None + and interval["low"] >= min_pass_rate + ): + disposition = DISPOSITION_POSITIVE + else: + disposition = DISPOSITION_UNDERPOWERED + return disposition, interval + +FIXTURE_CONFIG_OVERRIDES = { + "d_model": 64, + "denoiser_layers": 2, + "context_layers": 1, + "max_chain": 3, + "max_search_steps": 6, + "beam_width": 3, + "expand_per_state": 3, +} + + +@dataclass(frozen=True) +class ArmOutcome: + """Paired per-example final outcome for one arm (duck-types slm155's).""" + + record_id: str + arm_id: str + seed: int + final_source: str + v1_verdict: bool + hard_rank: int + commit_reason: str | None = None + + def to_dict(self) -> dict[str, Any]: + return { + "record_id": self.record_id, + "arm_id": self.arm_id, + "seed": self.seed, + "final_source": self.final_source, + "v1_verdict": self.v1_verdict, + "hard_rank": self.hard_rank, + "commit_reason": self.commit_reason, + } + + +def _inventory_for(source: str) -> list[str]: + import re + + slots = sorted(set(re.findall(r":[A-Za-z][\w.]*", source))) + return slots or [":content"] + + +def _prompt_for(decision_id: str) -> str: + return f"fixture decision {decision_id}: render the accepted content" + + +def _build_model(records, *, seed: int, value_label_mode: str, stop_accounting: str): + import torch + + from slm_training.models.tree_edit_diffusion import ( + TreeEditDiffusionConfig, + TreeEditDiffusionModel, + ) + + torch.manual_seed(seed) + config = TreeEditDiffusionConfig( + seed=seed, + value_label_mode=value_label_mode, + stop_slot_accounting=stop_accounting, + context_backend="scratch", + **FIXTURE_CONFIG_OVERRIDES, + ) + return TreeEditDiffusionModel.from_records(records, config=config, device="cpu") + + +def _train(model, records, *, steps: int, lr: float): + import torch + + optimizer = torch.optim.Adam(model.trainable_parameters(), lr=lr) + model.train() + for _ in range(steps): + loss = model.training_loss(records) + optimizer.zero_grad() + loss.backward() + optimizer.step() + model.eval() + return model + + +def run_screen( + *, + n_train: int, + n_eval: int, + seeds: tuple[int, ...], + steps: int, + lr: float, + scorer_steps: int, + decisions_out: Path, + power_rule: dict[str, float] | None = None, +) -> dict[str, Any]: + import torch + + from slm_training.dsl.schema import ExampleRecord + + torch.set_num_threads(1) # tiny CPU fixture; identical for every arm + start = time.perf_counter() + + train_decisions = make_fixture_decisions(n=n_train, seed=0) + eval_decisions = make_fixture_decisions(n=n_eval, seed=1) + scorer_result = train_fixture_scorer( + train_decisions, + config=LegalActionScorerConfig(variant="mlp", seed=0), + steps=scorer_steps, + lr=0.05, + ) + scorer = scorer_result["scorer"] + + def ar_source_for(decision) -> str: + scores = scorer.score( + decision.context_features, + decision.state_features, + decision.legal_actions, + plan_features=decision.plan_features, + plan_action_features=decision.plan_action_features, + pack_id=decision.pack_id, + ) + chosen = scorer.decode(scores, decision.legal_actions).action_identity + return _program_from_actions([chosen] if chosen else []) + + train_records = [ + ExampleRecord( + id=d.decision_id, + prompt=_prompt_for(d.decision_id), + openui=_program_from_actions( + [d.accepted_action_ids[0]] if d.accepted_action_ids else [] + ), + placeholders=_inventory_for( + _program_from_actions( + [d.accepted_action_ids[0]] if d.accepted_action_ids else [] + ) + ), + ) + for d in train_decisions + ] + + model_hist = _train( + _build_model( + train_records, + seed=0, + value_label_mode="mutation_count", + stop_accounting="legacy", + ), + train_records, + steps=steps, + lr=lr, + ) + model_imp = _train( + _build_model( + train_records, + seed=0, + value_label_mode="bounded_distance", + stop_accounting="corrected", + ), + train_records, + steps=steps, + lr=lr, + ) + + outcomes: list[ArmOutcome] = [] + commit_rows: list[CommitDecision] = [] + + def record(arm: str, record_id: str, seed: int, final: str, reason: str | None): + ev = hard_evidence(final) + outcomes.append( + ArmOutcome( + record_id=record_id, + arm_id=arm, + seed=seed, + final_source=final, + v1_verdict=ev.v1_verdict, + hard_rank=ev.rank, + commit_reason=reason, + ) + ) + + for seed in seeds: + for decision in eval_decisions: + rid = decision.decision_id + prompt = _prompt_for(rid) + ar_source = ar_source_for(decision) + inventory = _inventory_for(ar_source) + + # Arm 1: AR only. + record("ar_only", rid, seed, ar_source, None) + + # Arm 2: repair-only from the minimal seed (improved model decode). + ctx, ctx_pad = model_imp._encode_context( + [model_imp._format_context(prompt, slot_contract=inventory)] + ) + seed_final, _ = model_imp._decode_one(ctx, ctx_pad, inventory) + record("repair_only", rid, seed, seed_final, None) + + # Arm 3: AR → historical repair (pre-LAR2 value/stop config). + cand_hist, _ = repair_decode(model_hist, ar_source, inventory, prompt) + decision_hist = commit( + ar_source, + cand_hist, + evidence={ + "learned_score_source": state_value( + model_hist, ar_source, inventory, prompt + ), + "learned_score_candidate": state_value( + model_hist, cand_hist, inventory, prompt + ), + }, + record_id=rid, + arm_id="ar_repair_historical", + seed=seed, + ) + commit_rows.append(decision_hist) + record( + "ar_repair_historical", rid, seed, decision_hist.final, decision_hist.reason + ) + + # Arm 4: AR → improved repair (this branch's config). + cand_imp, _ = repair_decode(model_imp, ar_source, inventory, prompt) + decision_imp = commit( + ar_source, + cand_imp, + evidence={ + "learned_score_source": state_value( + model_imp, ar_source, inventory, prompt + ), + "learned_score_candidate": state_value( + model_imp, cand_imp, inventory, prompt + ), + }, + record_id=rid, + arm_id="ar_repair_improved", + seed=seed, + ) + commit_rows.append(decision_imp) + record( + "ar_repair_improved", rid, seed, decision_imp.final, decision_imp.reason + ) + + # Arm 5: oracle commit selector (non-promotable upper bound). + decision_oracle = oracle_commit( + ar_source, cand_imp, record_id=rid, arm_id="oracle_commit", seed=seed + ) + commit_rows.append(decision_oracle) + record("oracle_commit", rid, seed, decision_oracle.final, decision_oracle.reason) + + decisions_out.parent.mkdir(parents=True, exist_ok=True) + with decisions_out.open("w", encoding="utf-8") as fh: + for row in commit_rows: + fh.write(json.dumps(row.to_dict(), sort_keys=True) + "\n") + + by_arm: dict[str, list[ArmOutcome]] = {arm: [] for arm in ARMS} + for outcome in outcomes: + by_arm[outcome.arm_id].append(outcome) + + ar_outcomes = by_arm["ar_only"] + paired: dict[str, dict[str, int]] = {} + for arm in ARMS: + if arm == "ar_only": + continue + paired[arm] = paired_validity_table(ar_outcomes, by_arm[arm]) + + def valid_rate(arm: str) -> float: + rows = by_arm[arm] + return sum(o.v1_verdict for o in rows) / max(len(rows), 1) + + n_paired = sum(paired["ar_repair_improved"].values()) - paired["ar_repair_improved"]["unpaired"] + improvements = paired["ar_repair_improved"]["a_invalid_b_valid"] + damages = paired["ar_repair_improved"]["a_valid_b_invalid"] + wilson = wilson_interval(improvements, max(n_paired, 0)) + wilson_damage = wilson_interval(damages, max(n_paired, 0)) + + reachability = {p: Path(p).exists() for p in REACHABILITY_ARTIFACTS} + gates = { + "safety": { + "rule": PREREGISTERED["safety_rule"], + "invalid_over_valid": damages, + "pass": damages == 0, + }, + "value": { + "rule": PREREGISTERED["value_rule"], + "improvements": improvements, + "n_paired": n_paired, + "wilson": wilson, + "min_useful_effect": PREREGISTERED["min_useful_effect"], + "pass": bool( + wilson["low"] is not None + and wilson["low"] > PREREGISTERED["min_useful_effect"] + and damages == 0 + ), + }, + "reachability": { + "artifacts": reachability, + "pass": all(reachability.values()), + }, + } + + safety_pass = gates["safety"]["pass"] + value_pass = gates["value"]["pass"] + reach_pass = gates["reachability"]["pass"] + power_evidence: dict[str, Any] | None = None + per_seed_results: list[dict[str, Any]] | None = None + if power_rule is None: + if safety_pass and value_pass and reach_pass: + disposition = DISPOSITION_POSITIVE + elif not safety_pass or ( + wilson["high"] is not None + and wilson["high"] < PREREGISTERED["min_useful_effect"] + ): + disposition = DISPOSITION_NEGATIVE + else: + disposition = DISPOSITION_INCONCLUSIVE + else: + # SLM-431: per-seed pass/fail under the preregistered improvement + # rule (>=1 improvement, 0 damages per seed), then the Wilson power + # decision. Safety and reachability gates stay independent. + per_seed_results = [] + for seed in seeds: + seed_ar = [o for o in ar_outcomes if o.seed == seed] + seed_arm = [o for o in by_arm["ar_repair_improved"] if o.seed == seed] + seed_paired = paired_validity_table(seed_ar, seed_arm) + seed_improvements = seed_paired["a_invalid_b_valid"] + seed_damages = seed_paired["a_valid_b_invalid"] + per_seed_results.append( + { + "seed": seed, + "improvements": seed_improvements, + "damages": seed_damages, + "paired": seed_paired, + "pass": seed_improvements >= 1 and seed_damages == 0, + } + ) + passed_seeds = sum(row["pass"] for row in per_seed_results) + disposition, power_interval = power_disposition( + passed_seeds=passed_seeds, + n_seeds=len(seeds), + min_pass_rate=power_rule["min_pass_rate"], + safety_pass=safety_pass, + reachability_pass=reach_pass, + confidence_level=float(power_rule.get("confidence_level", 0.95)), + ) + power_evidence = { + "min_pass_rate": float(power_rule["min_pass_rate"]), + "seed_pass_rule": POWER_PREREGISTERED["seed_pass_rule"], + "disposition_rule": POWER_PREREGISTERED["disposition_rule"], + "passed_seed_count": passed_seeds, + "seed_count": len(seeds), + "interval": {"method": "wilson_score", **power_interval}, + } + + if power_rule is None: + lar3_statement = { + DISPOSITION_POSITIVE: ( + "LAR3 (learned repair integration) is OPEN: the do-no-harm hybrid " + "cleared safety, value, and reachability gates on the frozen screen." + ), + DISPOSITION_NEGATIVE: ( + "LAR3 (learned repair integration) is CLOSED: repair damaged valid " + "AR programs or its paired value was ruled out below the minimum " + "useful effect on the frozen screen." + ), + DISPOSITION_INCONCLUSIVE: ( + "LAR3 (learned repair integration) remains OPEN but NOT advanced: " + "the screen neither cleared the value gate nor ruled the effect " + "out at fixture power; a powered rerun is required before LAR3 " + "can open or close." + ), + }[disposition] + else: + lar3_statement = { + DISPOSITION_POSITIVE: ( + "LAR3 (learned repair integration): BOTH PR #853-#856 reopening " + "conditions are now met (SLM-421's recurrence-health condition " + "and this powered do-no-harm repair value gate). This artifact " + "does NOT reopen LAR3 — a separate, explicit LAR3-reopening " + "decision issue is required next." + ), + DISPOSITION_NEGATIVE: ( + "LAR3 (learned repair integration) stays CLOSED: the powered " + "rerun damaged valid AR programs or ruled out the preregistered " + "minimum per-seed pass rate; the second reopening condition is " + "not met." + ), + DISPOSITION_UNDERPOWERED: ( + "LAR3 (learned repair integration) remains unresolved: at this " + "seed count the Wilson interval neither clears nor rules out " + "the preregistered pass rate; a higher-powered rerun (within " + "MAX_RUN_MINUTES) is required before the second reopening " + "condition can resolve." + ), + }[disposition] + + payload = { + "experiment": EXPERIMENT_ID, + "issue": "SLM-317", + "question": ( + "Does a do-no-harm AR→repair hybrid (deterministic commit rule) " + "improve paired hard-valid outcomes over AR only without ever " + "damaging a valid AR program?" + ), + "preregistered": PREREGISTERED, + "arms": list(ARMS), + "promotable_arms": sorted(PROMOTABLE_ARMS), + "config": { + "n_train_decisions": n_train, + "n_eval_decisions": n_eval, + "seeds": list(seeds), + "steps": steps, + "lr": lr, + "scorer_steps": scorer_steps, + "fixture_config_overrides": FIXTURE_CONFIG_OVERRIDES, + "torch_num_threads": 1, + }, + "arm_summaries": { + arm: { + "n": len(by_arm[arm]), + "valid_rate": valid_rate(arm), + "mean_hard_rank": ( + sum(o.hard_rank for o in by_arm[arm]) / max(len(by_arm[arm]), 1) + ), + } + for arm in ARMS + }, + "paired_vs_ar_only": paired, + "invalid_over_valid_per_arm": { + arm: paired[arm]["a_valid_b_invalid"] for arm in paired + }, + "commit_reason_counts": { + arm: { + reason: sum(1 for o in by_arm[arm] if o.commit_reason == reason) + for reason in sorted( + {o.commit_reason for o in by_arm[arm] if o.commit_reason} + ) + } + for arm in ("ar_repair_historical", "ar_repair_improved", "oracle_commit") + }, + "gates": gates, + "wilson_damage": wilson_damage, + "oracle_upper_bound": { + "valid_rate": valid_rate("oracle_commit"), + "sanity": valid_rate("oracle_commit") >= valid_rate("ar_repair_improved"), + }, + "per_example_outcomes": [o.to_dict() for o in outcomes], + "decisions_path": str(decisions_out), + "disposition": disposition, + "lar3_statement": lar3_statement, + "wall_seconds": time.perf_counter() - start, + "honesty": ( + "Fixture-scale wiring screen: the frozen corpus is the SLM-155 " + "synthetic decision fixture (n eval decisions below any ship-gate " + "prompt count), models are tiny CPU fixtures trained for a handful " + "of steps, and no ship-gate claim is made. The commit rule is " + "deterministic and metamorphism-invariant by construction; the " + "screen measures the mechanism, not production quality. The " + "historical arm's pre-SLM-305 4-action space is declared " + "non-reproducible on this branch (see preregistered deviation)." + ), + } + payload["version_stamp"] = build_version_stamp( + "harness.experiments.slm317_repair_hybrid", + "harness.experiments.slm155_factorization_comparison", + "harness.experiments.slm299_edit_reachability", + ) + if power_evidence is not None: + payload["issue"] = "SLM-431" + payload["power_rerun_of"] = "SLM-317" + payload["power_rule"] = power_evidence + payload["per_seed_results"] = per_seed_results + payload["honesty"] = payload["honesty"] + ( + " SLM-431 powered rerun: only the seed count and the value " + "decision rule changed (per-seed pass/fail + Wilson interval, " + "preregistered and locked before any new seed was read); the " + "corpus, matched arms, do-no-harm commit rule, safety gate, and " + "reachability gate are byte-identical to SLM-317. New seeds are " + "disjoint from SLM-317's original [0, 1]." + ) + return payload + + +def render_markdown(payload: dict) -> str: + def f(x: float) -> str: + return f"{x:.3f}" + + lines = [ + "# SLM-317 (LAR2-06): do-no-harm AR→repair hybrid screen", + "", + f"**Disposition: `{payload['disposition']}`** — {payload['lar3_statement']}", + "", + "Fixture-scale mechanism screen; **not a ship claim**.", + "", + "## Preregistered (locked before results)", + "", + f"- safety: {payload['preregistered']['safety_rule']}", + f"- value: {payload['preregistered']['value_rule']}", + f"- disposition: {payload['preregistered']['disposition_rule']}", + f"- declared deviation: {payload['preregistered']['historical_deviation']}", + "", + "## Arms (matched examples + seeds + budgets)", + "", + "| arm | n | hard-valid rate | mean hard rank | invalid-over-valid vs AR |", + "| --- | --- | --- | --- | --- |", + ] + iov = payload["invalid_over_valid_per_arm"] + for arm in payload["arms"]: + s = payload["arm_summaries"][arm] + lines.append( + f"| {arm} | {s['n']} | {f(s['valid_rate'])} | " + f"{f(s['mean_hard_rank'])} | {iov.get(arm, '—')} |" + ) + lines += [ + "", + "## Paired outcomes vs `ar_only` (per-example, nothing aggregated away)", + "", + "| arm | both valid | AR valid, arm invalid | AR invalid, arm valid | both invalid | unpaired |", + "| --- | --- | --- | --- | --- | --- |", + ] + for arm, counts in payload["paired_vs_ar_only"].items(): + lines.append( + f"| {arm} | {counts['both_valid']} | {counts['a_valid_b_invalid']} | " + f"{counts['a_invalid_b_valid']} | {counts['both_invalid']} | {counts['unpaired']} |" + ) + gates = payload["gates"] + wilson = gates["value"]["wilson"] + lines += [ + "", + "## Advancement gates", + "", + f"- **Safety**: invalid-over-valid = {gates['safety']['invalid_over_valid']} " + f"(must be 0) → {'PASS' if gates['safety']['pass'] else 'FAIL'}", + f"- **Value**: improvements {gates['value']['improvements']}/{gates['value']['n_paired']}, " + f"Wilson 95% [{wilson['low']}, {wilson['high']}] vs minimum useful effect " + f"{gates['value']['min_useful_effect']} → {'PASS' if gates['value']['pass'] else 'FAIL'}", + f"- **Reachability/provenance**: slm299/291 artifacts present → " + f"{'PASS' if gates['reachability']['pass'] else 'FAIL'}", + ] + power = payload.get("power_rule") + if power is not None: + interval = power["interval"] + lines += [ + "", + "## Preregistration (powered rerun, locked before any new seed was read)", + "", + f"- seed pass rule: {power['seed_pass_rule']}", + f"- disposition rule: {power['disposition_rule']}", + f"- seed count: `{power['seed_count']}` " + f"(disjoint from SLM-317's original [0, 1])", + f"- observed passing seeds: `{power['passed_seed_count']}`", + f"- observed pass rate: `{interval['estimate']:.4f}` " + f"(Wilson 95% CI [`{interval['low']:.4f}`, `{interval['high']:.4f}`], " + f"n=`{interval['n']}`) vs preregistered min pass rate " + f"`{power['min_pass_rate']}`", + "", + "| seed | improvements | damages | pass |", + "| ---: | ---: | ---: | --- |", + ] + for row in payload["per_seed_results"]: + lines.append( + f"| {row['seed']} | {row['improvements']} | {row['damages']} | " + f"{'PASS' if row['pass'] else 'FAIL'} |" + ) + lines += [ + "", + "## Commit reasons", + "", + "| arm | reason | count |", + "| --- | --- | --- |", + ] + for arm, reasons in payload["commit_reason_counts"].items(): + for reason, count in reasons.items(): + lines.append(f"| {arm} | {reason} | {count} |") + lines += [ + "", + f"Oracle commit upper bound hard-valid rate: " + f"{f(payload['oracle_upper_bound']['valid_rate'])} " + f"(sanity ≥ improved hybrid: {payload['oracle_upper_bound']['sanity']}).", + "", + f"Durable per-example commit decisions: `{payload['decisions_path']}`.", + "", + "## Honesty", + "", + payload["honesty"], + "", + ] + return "\n".join(lines) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--steps", type=int, default=6) + parser.add_argument("--lr", type=float, default=3e-3) + parser.add_argument("--n-train", type=int, default=16) + parser.add_argument("--n-eval", type=int, default=8) + parser.add_argument("--seeds", type=int, nargs="+", default=[0, 1]) + parser.add_argument("--scorer-steps", type=int, default=20) + parser.add_argument( + "--min-pass-rate", + type=float, + default=None, + help=( + "SLM-431 powered-rerun mode (default off). Enables a " + "Wilson-interval decision rule over the per-seed pass rate (see " + "slm_training.evals.power_protocol.wilson_interval) instead of " + "the original 2-seed per-example screen: repair_positive iff the " + "interval lower bound clears this rate (with safety AND " + "reachability passing), repair_negative iff the upper bound " + "falls below it, else inconclusive_underpowered. Required for " + "any --seeds other than the original [0, 1]; new seeds must be " + "disjoint from [0, 1]." + ), + ) + parser.add_argument("--json-out", type=Path, default=DEFAULT_JSON_OUT) + parser.add_argument("--md-out", type=Path, default=DEFAULT_MD_OUT) + parser.add_argument("--decisions-out", type=Path, default=DEFAULT_DECISIONS_OUT) + args = parser.parse_args(argv) + if args.seeds != [0, 1] and args.min_pass_rate is None: + parser.error( + "--seeds other than the original [0, 1] requires --min-pass-rate " + "(a raw per-example screen is not a meaningful decision rule at " + "arbitrary seed counts)" + ) + + power_rule = ( + None + if args.min_pass_rate is None + else {"min_pass_rate": args.min_pass_rate} + ) + json_out: Path = args.json_out + md_out: Path = args.md_out + decisions_out: Path = args.decisions_out + if power_rule is not None: + # Never clobber the frozen SLM-317 20260724 record: powered runs get + # their own dated stem (mirrors SLM-421's powered-rerun stem switch). + today = _today_slug() + if json_out == DEFAULT_JSON_OUT: + json_out = Path( + f"docs/design/iter-slm317-repair-hybrid-powered-rerun-{today}.json" + ) + if md_out == DEFAULT_MD_OUT: + md_out = Path( + f"docs/design/iter-slm317-repair-hybrid-powered-rerun-{today}.md" + ) + if decisions_out == DEFAULT_DECISIONS_OUT: + decisions_out = Path( + f"outputs/slm317/commit_decisions_powered_{today}.jsonl" + ) + + payload = run_screen( + n_train=args.n_train, + n_eval=args.n_eval, + seeds=tuple(args.seeds), + steps=args.steps, + lr=args.lr, + scorer_steps=args.scorer_steps, + decisions_out=decisions_out, + power_rule=power_rule, + ) + json_out.parent.mkdir(parents=True, exist_ok=True) + json_out.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + md_out.write_text(render_markdown(payload), encoding="utf-8") + print( + f"disposition={payload['disposition']} " + f"safety={payload['gates']['safety']['pass']} " + f"value={payload['gates']['value']['pass']} " + f"iov={payload['invalid_over_valid_per_arm'].get('ar_repair_improved')} " + f"-> {json_out}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/slm_training/harnesses/experiments/slm317_repair_hybrid.py b/src/slm_training/harnesses/experiments/slm317_repair_hybrid.py new file mode 100644 index 000000000..2634f176d --- /dev/null +++ b/src/slm_training/harnesses/experiments/slm317_repair_hybrid.py @@ -0,0 +1,446 @@ +"""SLM-317 (LAR2-06): do-no-harm AR→repair hybrid commit machinery. + +A valid-state repair arm is only useful if it never damages a correct AR +program. This module provides the deterministic, metamorphism-invariant +commit rule for the AR→repair hybrid screen: + +- :func:`hard_evidence` scores one final-program source on the LAR1-02 + unified ladder: official parse (``dsl.parser.validate``) → output contract + (``output_contract_violations``) → ``meaningful_program_v1`` verdict. The + ladder rank is the ONLY hard authority. +- :func:`commit` applies the preregistered do-no-harm rule: + + a. hard evidence improves → COMMIT (``hard_improvement``); + b. hard evidence regresses → RETAIN source (``hard_regression``); + c. hard evidence unchanged and a calibrated learned score (model value or + oracle distance improvement) improves → COMMIT + (``soft_improvement_no_hard_regression``); + d. otherwise RETAIN (``no_improvement``); a missing/empty candidate is an + ABSTAIN (``candidate_unavailable``), never a silent commit. + +- :func:`oracle_commit` is the non-promotable upper bound: commit exactly + when the candidate's hard evidence is strictly better. +- Metamorphic transforms (:func:`alpha_rename`, :func:`reorder_statements`, + :func:`normalize_formatting`, :func:`ast_roundtrip`) are + semantics-preserving source rewrites used by the invariance tests: hard + evidence and commit decisions must be identical across them. +- :func:`repair_decode` runs the branch's value-guided beam repair starting + from an arbitrary valid source state (not the minimal seed), fail-closed + via ``TreeEditSpace.apply``. + +The module is torch-free at import time; ``repair_decode``/``state_value`` +import torch lazily through the model they are given. +""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass, field +from typing import Any + +from slm_training.dsl.language_contract import output_contract_violations +from slm_training.dsl.parser import validate +from slm_training.harnesses.model_build.eval_runner import ( + meaningful_program_v1_report, +) +from slm_training.models.tree_edit_diffusion import ( + ACTION_STOP, + Statement, + parse_statements, + render_statements, +) + +EXPERIMENT_ID = "slm317-repair-hybrid" +COMMIT_DECISION_SCHEMA = "slm317_commit_decision/v1" + +# Commit actions. +ACTION_COMMIT = "commit" +ACTION_RETAIN = "retain" +ACTION_ABSTAIN = "abstain" + +# Preregistered reason codes (the full closed set). +REASON_HARD_IMPROVEMENT = "hard_improvement" +REASON_HARD_REGRESSION = "hard_regression" +REASON_SOFT_IMPROVEMENT = "soft_improvement_no_hard_regression" +REASON_NO_IMPROVEMENT = "no_improvement" +REASON_CANDIDATE_UNAVAILABLE = "candidate_unavailable" +REASON_ORACLE_HARD_IMPROVEMENT = "oracle_hard_improvement" +REASON_ORACLE_NO_GAIN = "oracle_no_gain" + +# Dispositions (exactly one per screen run). +DISPOSITION_POSITIVE = "repair_positive" +DISPOSITION_NEGATIVE = "repair_negative" +DISPOSITION_INCONCLUSIVE = "inconclusive" + + +# --------------------------------------------------------------------------- # +# Hard evidence (LAR1-02 ladder) +# --------------------------------------------------------------------------- # + + +@dataclass(frozen=True) +class HardEvidence: + """Deterministic verifier/contract evidence for one final program.""" + + parse_ok: bool + contract_ok: bool + v1_verdict: bool + v1_reason_codes: tuple[str, ...] + contract_violations: tuple[str, ...] + + @property + def rank(self) -> int: + """Ordered ladder: 0 unparseable < 1 parsed < 2 contract-clean < 3 v1-valid.""" + if not self.parse_ok: + return 0 + if not self.contract_ok: + return 1 + if not self.v1_verdict: + return 2 + return 3 + + def to_dict(self) -> dict[str, Any]: + data = asdict(self) + data["v1_reason_codes"] = list(self.v1_reason_codes) + data["contract_violations"] = list(self.contract_violations) + data["rank"] = self.rank + return data + + +def hard_evidence(source: str | None) -> HardEvidence: + """Score one program source on the deterministic ladder. Never raises.""" + if not source: + return HardEvidence(False, False, False, ("no_program",), ()) + parse_ok = False + try: + validate(source) + parse_ok = True + except Exception: # noqa: BLE001 - parse fact, never raised + parse_ok = False + violations: tuple[str, ...] = () + if parse_ok: + violations = tuple(output_contract_violations(source)) + contract_ok = parse_ok and not violations + report = meaningful_program_v1_report(source, gold=None) + return HardEvidence( + parse_ok=parse_ok, + contract_ok=contract_ok, + v1_verdict=bool(report["verdict"]), + v1_reason_codes=tuple(report.get("reason_codes", ())), + contract_violations=violations, + ) + + +# --------------------------------------------------------------------------- # +# Commit rule +# --------------------------------------------------------------------------- # + + +@dataclass(frozen=True) +class CommitDecision: + """Durable per-example commit record (source + candidate + exact reason).""" + + action: str # commit | retain | abstain + reason: str + source: str + candidate: str + final: str + source_evidence: HardEvidence + candidate_evidence: HardEvidence + learned_score_source: float | None = None + learned_score_candidate: float | None = None + record_id: str = "" + arm_id: str = "" + seed: int = 0 + notes: list[str] = field(default_factory=list) + + def to_dict(self) -> dict[str, Any]: + return { + "schema": COMMIT_DECISION_SCHEMA, + "action": self.action, + "reason": self.reason, + "source": self.source, + "candidate": self.candidate, + "final": self.final, + "source_evidence": self.source_evidence.to_dict(), + "candidate_evidence": self.candidate_evidence.to_dict(), + "learned_score_source": self.learned_score_source, + "learned_score_candidate": self.learned_score_candidate, + "record_id": self.record_id, + "arm_id": self.arm_id, + "seed": self.seed, + "notes": list(self.notes), + } + + +def commit( + source: str, + candidate: str | None, + *, + evidence: dict[str, Any] | None = None, + record_id: str = "", + arm_id: str = "", + seed: int = 0, +) -> CommitDecision: + """Do-no-harm commit rule (deterministic; see module docstring). + + ``evidence`` may carry ``learned_score_source`` / + ``learned_score_candidate`` (model value or oracle distance improvement; + higher = better) and ``learned_margin`` (strict improvement margin, + default 0.0). Learned scores are only consulted when hard evidence is + unchanged, and a soft commit is forbidden whenever hard evidence regresses. + """ + evidence = evidence or {} + src_ev = hard_evidence(source) + if candidate is None or not candidate.strip(): + return CommitDecision( + action=ACTION_ABSTAIN, + reason=REASON_CANDIDATE_UNAVAILABLE, + source=source, + candidate="", + final=source, + source_evidence=src_ev, + candidate_evidence=hard_evidence(None), + record_id=record_id, + arm_id=arm_id, + seed=seed, + ) + cand_ev = hard_evidence(candidate) + soft_s = evidence.get("learned_score_source") + soft_c = evidence.get("learned_score_candidate") + margin = float(evidence.get("learned_margin", 0.0)) + + if cand_ev.rank > src_ev.rank: + action, reason, final = ACTION_COMMIT, REASON_HARD_IMPROVEMENT, candidate + elif cand_ev.rank < src_ev.rank: + action, reason, final = ACTION_RETAIN, REASON_HARD_REGRESSION, source + elif ( + soft_s is not None + and soft_c is not None + and float(soft_c) > float(soft_s) + margin + ): + action, reason, final = ACTION_COMMIT, REASON_SOFT_IMPROVEMENT, candidate + else: + action, reason, final = ACTION_RETAIN, REASON_NO_IMPROVEMENT, source + return CommitDecision( + action=action, + reason=reason, + source=source, + candidate=candidate, + final=final, + source_evidence=src_ev, + candidate_evidence=cand_ev, + learned_score_source=None if soft_s is None else float(soft_s), + learned_score_candidate=None if soft_c is None else float(soft_c), + record_id=record_id, + arm_id=arm_id, + seed=seed, + ) + + +def oracle_commit( + source: str, + candidate: str | None, + *, + record_id: str = "", + arm_id: str = "", + seed: int = 0, +) -> CommitDecision: + """Non-promotable upper bound: commit exactly on strict hard improvement.""" + if candidate is None or not candidate.strip(): + return commit( + source, candidate, record_id=record_id, arm_id=arm_id, seed=seed + ) + src_ev = hard_evidence(source) + cand_ev = hard_evidence(candidate) + better = cand_ev.rank > src_ev.rank + return CommitDecision( + action=ACTION_COMMIT if better else ACTION_RETAIN, + reason=REASON_ORACLE_HARD_IMPROVEMENT if better else REASON_ORACLE_NO_GAIN, + source=source, + candidate=candidate, + final=candidate if better else source, + source_evidence=src_ev, + candidate_evidence=cand_ev, + record_id=record_id, + arm_id=arm_id, + seed=seed, + notes=["oracle selector; non-promotable upper bound"], + ) + + +# --------------------------------------------------------------------------- # +# Metamorphic transforms (semantics-preserving; used by invariance tests) +# --------------------------------------------------------------------------- # + + +def alpha_rename(source: str, *, prefix: str = "m") -> str: + """Consistently rename non-root node identifiers (references included).""" + statements = parse_statements(source) + if statements is None: + return source + mapping: dict[str, str] = {} + for stmt in statements: + if stmt.name != "root": + mapping[stmt.name] = f"{prefix}{len(mapping)}" + renamed = [ + Statement( + name=mapping.get(stmt.name, stmt.name), + comp=stmt.comp, + children=[mapping.get(c, c) for c in stmt.children], + rest=stmt.rest, + has_list=stmt.has_list, + ) + for stmt in statements + ] + return render_statements(renamed) + + +def reorder_statements(source: str) -> str: + """Move the root statement last (declarations first); semantics preserved.""" + statements = parse_statements(source) + if statements is None: + return source + roots = [s for s in statements if s.name == "root"] + others = [s for s in statements if s.name != "root"] + return render_statements(others + roots) + + +def normalize_formatting(source: str) -> str: + """Re-render the structural form (canonical spacing/newlines).""" + statements = parse_statements(source) + if statements is None: + return source.strip() + return render_statements(statements) + + +def ast_roundtrip(source: str) -> str: + """Equivalent AST serialization via the official parser's serialized form.""" + try: + program = validate(source) + except Exception: # noqa: BLE001 - unparseable input round-trips to itself + return source + return program.serialized or source.strip() + + +# --------------------------------------------------------------------------- # +# Repair decode from an arbitrary valid source state +# --------------------------------------------------------------------------- # + + +def repair_decode( + model: Any, + source: str, + inventory: list[str], + prompt: str, +) -> tuple[str, dict[str, Any]]: + """Value-guided beam repair seeded from ``source`` (not the minimal seed). + + Mirrors ``TreeEditDiffusionModel._decode_one`` (same fail-closed + ``TreeEditSpace.apply``, same value re-scoring, same STOP handling) with + the beam initialized on the caller's valid state. An unparseable or + invalid source fails closed: returns ``("", {"failure": ...})``. + """ + import torch # lazy: module import stays torch-free + + statements = parse_statements(source) + if statements is None: + return "", {"failure": "source_unparseable", "kind": "tree_edit_repair"} + from slm_training.models.tree_edit_diffusion import _is_valid + + if not _is_valid(source): + return "", {"failure": "source_invalid", "kind": "tree_edit_repair"} + + model.eval() + ctx, ctx_pad = model._encode_context( + [model._format_context(prompt, slot_contract=list(inventory))] + ) + cfg = model.config + beam: list[tuple[float, list[Statement], bool]] = [(0.0, statements, False)] + evidence: dict[str, Any] = {"steps": 0, "expansions": 0, "kind": "tree_edit_repair"} + with torch.no_grad(): + for _ in range(cfg.max_search_steps): + live = [entry for entry in beam if not entry[2]] + if not live: + break + sources = [render_statements(s) for _, s, _ in live] + out = model.policy( + model._state_batch(sources), + model.tokenizer.pad_id, + ctx.expand(len(sources), -1, -1), + ctx_pad.expand(len(sources), -1), + ) + next_beam = [entry for entry in beam if entry[2]] + seen: set[str] = { + render_statements(s) for _, s, frozen in next_beam if frozen + } + for row, (_, stmts, _) in enumerate(live): + candidates = model._enumerate_edits(out, row, len(stmts), len(inventory)) + expanded = 0 + for score, edit in candidates: + if expanded >= cfg.expand_per_state: + break + if edit.action == ACTION_STOP: + text = render_statements(stmts) + if text in seen: + continue + seen.add(text) + next_beam.append((float(out["value"][row]), stmts, True)) + expanded += 1 + continue + child = model.space.apply(stmts, edit, inventory, reason=[]) + if child is None: + continue + text = render_statements(child) + if text in seen: + continue + seen.add(text) + next_beam.append((float(out["value"][row]), child, False)) + expanded += 1 + evidence["expansions"] += 1 + if not next_beam: + break + unfrozen = [entry for entry in next_beam if not entry[2]] + if unfrozen: + rescore = model.policy( + model._state_batch([render_statements(s) for _, s, _ in unfrozen]), + model.tokenizer.pad_id, + ctx.expand(len(unfrozen), -1, -1), + ctx_pad.expand(len(unfrozen), -1), + ) + rescored = [ + (float(rescore["value"][i]), entry[1], False) + for i, entry in enumerate(unfrozen) + ] + else: + rescored = [] + frozen = [entry for entry in next_beam if entry[2]] + beam = sorted(frozen + rescored, key=lambda e: e[0], reverse=True)[ + : cfg.beam_width + ] + evidence["steps"] += 1 + if all(entry[2] for entry in beam): + break + best = max(beam, key=lambda e: e[0]) + evidence["value"] = float(best[0]) + evidence["frozen"] = bool(best[2]) + return render_statements(best[1]), evidence + + +def state_value(model: Any, source: str, inventory: list[str], prompt: str) -> float | None: + """Calibrated learned score (policy value head) for one state; None if unparseable.""" + import torch # lazy + + if parse_statements(source) is None: + return None + model.eval() + ctx, ctx_pad = model._encode_context( + [model._format_context(prompt, slot_contract=list(inventory))] + ) + with torch.no_grad(): + out = model.policy( + model._state_batch([source]), + model.tokenizer.pad_id, + ctx, + ctx_pad, + ) + return float(out["value"][0]) diff --git a/src/slm_training/resources/versions.json b/src/slm_training/resources/versions.json index bbefb6afd..2ae1ce69a 100644 --- a/src/slm_training/resources/versions.json +++ b/src/slm_training/resources/versions.json @@ -2520,7 +2520,7 @@ ] }, "harness.experiments": { - "version": "v137", + "version": "v138", "kind": "harness", "paths": [ "scripts/run_slm298_capacity_context_curriculum.py", @@ -2532,6 +2532,11 @@ "tests/test_harnesses/experiments/test_locked_manifest_digest_threading.py" ], "history": [ + { + "version": "v138", + "date": "2026-07-27", + "note": "SLM-431 (LAR0-06): land src/slm_training/harnesses/experiments/slm317_repair_hybrid.py byte-identical from unmerged branch commit 48e5cadc (its own tests pass unmodified against main HEAD); the harness module itself carries no logic changes \u2014 the powered-rerun surface lives in scripts/run_slm317_repair_hybrid.py under harness.experiments.slm317_repair_hybrid v2." + }, { "version": "v137", "date": "2026-07-27", @@ -10838,6 +10843,29 @@ "note": "SLM-420 (DSH5-12): claim-separated advanced-operator disposition binding eleven independent verdicts (selector correctness, bulk atomicity, crossover work, transaction contracts/execution, set-valued selection, sequence merge, control-plane execution/learning, adaptive routing, event memory, parameterized templates, systems efficiency), each split across runtime-correctness/learned-benefit/partial-coverage-safety/systems-efficiency dimensions, plus the DSH3-33/SLM-408 inherited empty learned-policy inventory and CAP0/CAP1/CAP2 retention" } ] + }, + "harness.experiments.slm317_repair_hybrid": { + "version": "v2", + "kind": "harness", + "paths": [ + "src/slm_training/harnesses/experiments/slm317_repair_hybrid.py", + "scripts/run_slm317_repair_hybrid.py", + "tests/test_harnesses/experiments/test_slm317_repair_hybrid.py", + "docs/design/iter-slm317-repair-hybrid-20260724.json", + "docs/design/iter-slm317-repair-hybrid-20260724.md" + ], + "history": [ + { + "version": "v2", + "date": "2026-07-27", + "note": "SLM-431 (LAR0-06): additive, default-off powered-rerun surface on scripts/run_slm317_repair_hybrid.py mirroring SLM-421's run_slm138 power_rule: --min-pass-rate enables a preregistered per-seed pass/fail Wilson-interval decision (repair_positive / repair_negative / inconclusive_underpowered), locked before any new seed's results are read; power_rule=None preserves the original 2-seed behavior byte-for-byte. Do-no-harm commit rule, frozen SLM-155 corpus, matched arms, safety and reachability gates unchanged. Harness module landed byte-identical from 48e5cadc." + }, + { + "version": "v1", + "date": "2026-07-24", + "note": "initial registration; SLM-317 LAR2-06 do-no-harm AR->repair hybrid: metamorphism-invariant commit rule (hard ladder + calibrated soft score), matched arms incl. historical/improved repair + oracle upper bound, invalid-over-valid visibility, advancement gates and repair_positive|repair_negative|inconclusive disposition" + } + ] } } } diff --git a/tests/test_harnesses/experiments/test_slm317_repair_hybrid.py b/tests/test_harnesses/experiments/test_slm317_repair_hybrid.py new file mode 100644 index 000000000..2464b5e4f --- /dev/null +++ b/tests/test_harnesses/experiments/test_slm317_repair_hybrid.py @@ -0,0 +1,387 @@ +"""SLM-317 (LAR2-06): metamorphic invariance, commit truth table, visibility. + +Runs before any model evaluation: the commit/score pipeline must be invariant +to alpha-renaming, semantics-preserving reordering, formatting normalization, +and equivalent AST serializations — and the do-no-harm rule must cover its +full truth table deterministically. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from slm_training.harnesses.experiments.slm155_factorization_comparison import ( + paired_validity_table, +) +from slm_training.harnesses.experiments.slm317_repair_hybrid import ( + ACTION_ABSTAIN, + ACTION_COMMIT, + ACTION_RETAIN, + REASON_CANDIDATE_UNAVAILABLE, + REASON_HARD_IMPROVEMENT, + REASON_HARD_REGRESSION, + REASON_NO_IMPROVEMENT, + REASON_ORACLE_NO_GAIN, + REASON_SOFT_IMPROVEMENT, + alpha_rename, + ast_roundtrip, + commit, + hard_evidence, + normalize_formatting, + oracle_commit, + reorder_statements, +) +from scripts.run_slm317_repair_hybrid import ArmOutcome + +VALID = 'root = Stack([n0], "column")\nn0 = TextContent(":content")' +VALID_TWO = ( + 'root = Stack([n0, n1], "column")\n' + 'n0 = TextContent(":content")\n' + 'n1 = Button(":cta")' +) +INVALID_UNPARSEABLE = "root = Stack(([n0]," +VALID_PARSE_ONLY = 'root = Stack([], "column")' # parses; v1 invalid (empty stack) + +TRANSFORMS = (alpha_rename, reorder_statements, normalize_formatting, ast_roundtrip) + + +# --------------------------------------------------------------------------- # +# Metamorphic invariance (all four transforms) +# --------------------------------------------------------------------------- # + + +@pytest.mark.parametrize("source", [VALID, VALID_TWO, VALID_PARSE_ONLY]) +def test_hard_evidence_invariant_across_transforms(source: str) -> None: + base = hard_evidence(source) + for transform in TRANSFORMS: + variant = transform(source) + ev = hard_evidence(variant) + assert ev.rank == base.rank, transform.__name__ + assert ev.parse_ok == base.parse_ok + assert ev.contract_ok == base.contract_ok + assert ev.v1_verdict == base.v1_verdict + + +def test_commit_decision_invariant_across_transforms() -> None: + evidence = {"learned_score_source": 0.2, "learned_score_candidate": 0.9} + base = commit(VALID, VALID_TWO, evidence=evidence) + for transform in TRANSFORMS: + variant = commit(transform(VALID), transform(VALID_TWO), evidence=evidence) + assert variant.action == base.action, transform.__name__ + assert variant.reason == base.reason, transform.__name__ + assert variant.source_evidence.rank == base.source_evidence.rank + assert variant.candidate_evidence.rank == base.candidate_evidence.rank + + +def test_transforms_preserve_validity() -> None: + for transform in TRANSFORMS: + assert hard_evidence(transform(VALID)).v1_verdict, transform.__name__ + + +# --------------------------------------------------------------------------- # +# Commit-rule truth table (all branches + determinism) +# --------------------------------------------------------------------------- # + + +def test_truth_table_hard_improvement() -> None: + decision = commit(VALID_PARSE_ONLY, VALID) + assert decision.action == ACTION_COMMIT + assert decision.reason == REASON_HARD_IMPROVEMENT + assert decision.final == VALID + + +def test_truth_table_hard_regression_retains_source() -> None: + decision = commit(VALID, INVALID_UNPARSEABLE) + assert decision.action == ACTION_RETAIN + assert decision.reason == REASON_HARD_REGRESSION + assert decision.final == VALID + + +def test_truth_table_soft_improvement_no_hard_regression() -> None: + candidate = alpha_rename(VALID) # same hard rank, different source + decision = commit( + VALID, + candidate, + evidence={"learned_score_source": 0.1, "learned_score_candidate": 0.9}, + ) + assert decision.action == ACTION_COMMIT + assert decision.reason == REASON_SOFT_IMPROVEMENT + assert decision.final == candidate + + +def test_soft_commit_forbidden_when_hard_regresses() -> None: + decision = commit( + VALID, + INVALID_UNPARSEABLE, + evidence={"learned_score_source": 0.1, "learned_score_candidate": 0.9}, + ) + assert decision.action == ACTION_RETAIN + assert decision.reason == REASON_HARD_REGRESSION + + +def test_truth_table_no_improvement() -> None: + candidate = alpha_rename(VALID) + decision = commit(VALID, candidate) # no learned scores, same rank + assert decision.action == ACTION_RETAIN + assert decision.reason == REASON_NO_IMPROVEMENT + assert decision.final == VALID + + +def test_soft_margin_not_exceeded_retains() -> None: + candidate = alpha_rename(VALID) + decision = commit( + VALID, + candidate, + evidence={ + "learned_score_source": 0.5, + "learned_score_candidate": 0.5, + "learned_margin": 0.0, + }, + ) + assert decision.action == ACTION_RETAIN + assert decision.reason == REASON_NO_IMPROVEMENT + + +def test_truth_table_abstain_on_missing_candidate() -> None: + for candidate in (None, "", " "): + decision = commit(VALID, candidate) + assert decision.action == ACTION_ABSTAIN + assert decision.reason == REASON_CANDIDATE_UNAVAILABLE + assert decision.final == VALID + + +def test_commit_deterministic() -> None: + evidence = {"learned_score_source": 0.1, "learned_score_candidate": 0.9} + first = commit(VALID, VALID_TWO, evidence=evidence).to_dict() + second = commit(VALID, VALID_TWO, evidence=evidence).to_dict() + assert first == second + + +def test_decision_preserves_source_candidate_reason() -> None: + decision = commit(VALID_PARSE_ONLY, VALID, record_id="r1", arm_id="a", seed=3) + row = decision.to_dict() + assert row["source"] == VALID_PARSE_ONLY + assert row["candidate"] == VALID + assert row["reason"] == REASON_HARD_IMPROVEMENT + assert row["record_id"] == "r1" + assert row["schema"] == "slm317_commit_decision/v1" + json.dumps(row) # durable JSONL-serializable + + +# --------------------------------------------------------------------------- # +# Oracle selector upper bound +# --------------------------------------------------------------------------- # + + +def test_oracle_commits_only_on_strict_hard_improvement() -> None: + better = oracle_commit(VALID_PARSE_ONLY, VALID) + assert better.action == ACTION_COMMIT + worse = oracle_commit(VALID, VALID_PARSE_ONLY) + assert worse.action == ACTION_RETAIN + assert worse.reason == REASON_ORACLE_NO_GAIN + equal = oracle_commit(VALID, alpha_rename(VALID)) + assert equal.action == ACTION_RETAIN # equal rank is not an improvement + + +def test_oracle_upper_bound_dominates_rule() -> None: + # Whatever the deterministic rule commits, the oracle commits a superset + # (it commits on every strict hard improvement; the rule adds soft commits + # only when hard evidence is unchanged, which can never beat the oracle's + # final rank). + cases = [ + (VALID_PARSE_ONLY, VALID), + (VALID, VALID_PARSE_ONLY), + (VALID, INVALID_UNPARSEABLE), + (VALID, alpha_rename(VALID)), + ] + for source, candidate in cases: + rule = commit( + source, + candidate, + evidence={"learned_score_source": 0.0, "learned_score_candidate": 1.0}, + ) + oracle = oracle_commit(source, candidate) + assert oracle.source_evidence.rank <= oracle.candidate_evidence.rank or ( + oracle.action == ACTION_RETAIN + ) + assert hard_evidence(oracle.final).rank >= hard_evidence(rule.final).rank + + +# --------------------------------------------------------------------------- # +# Invalid-over-valid visibility + arm isolation +# --------------------------------------------------------------------------- # + + +def _outcome(record_id: str, arm: str, seed: int, verdict: bool) -> ArmOutcome: + return ArmOutcome( + record_id=record_id, + arm_id=arm, + seed=seed, + final_source=VALID if verdict else INVALID_UNPARSEABLE, + v1_verdict=verdict, + hard_rank=3 if verdict else 0, + ) + + +def test_invalid_over_valid_damage_surfaces() -> None: + """AR final valid + repaired final invalid must be visible, never hidden.""" + ar = [_outcome("r0", "ar_only", 0, True), _outcome("r1", "ar_only", 0, True)] + repaired = [ + _outcome("r0", "ar_repair_improved", 0, True), + _outcome("r1", "ar_repair_improved", 0, False), # damaged + ] + counts = paired_validity_table(ar, repaired) + assert counts["a_valid_b_invalid"] == 1 # the damage count is explicit + assert counts["both_valid"] == 1 + + +def test_commit_rule_makes_damage_unreachable() -> None: + """With the commit rule in the loop, a damaging candidate is retained away.""" + decision = commit(VALID, INVALID_UNPARSEABLE) + final_ev = hard_evidence(decision.final) + assert final_ev.v1_verdict # final is the retained valid AR source + + +def test_arm_isolation() -> None: + """Outcomes from one arm never leak into another arm's pairing.""" + ar = [_outcome("r0", "ar_only", 0, True)] + other = [_outcome("r0", "repair_only", 0, False)] + counts = paired_validity_table(ar, other) + assert counts["a_valid_b_invalid"] == 1 + # A third arm with no matching record contributes only unpaired rows. + third = [_outcome("rX", "ar_repair_improved", 0, True)] + counts2 = paired_validity_table(ar, third) + assert counts2["unpaired"] == 2 + assert counts2["a_valid_b_invalid"] == 0 + + +def test_paired_table_deterministic() -> None: + ar = [_outcome(f"r{i}", "ar_only", 0, i % 2 == 0) for i in range(4)] + arm = [_outcome(f"r{i}", "ar_repair_improved", 0, True) for i in range(4)] + assert paired_validity_table(ar, arm) == paired_validity_table(ar, arm) + + +# --------------------------------------------------------------------------- +# SLM-431 (LAR0-06): powered-rerun Wilson power rule (additive, default-off). +# Mirrors SLM-421's power_rule test additions for the SLM-282 recurrence +# screen: power_rule=None preserves the original 2-seed behavior byte-for- +# byte; the helper alone owns the new per-seed Wilson disposition. +# --------------------------------------------------------------------------- + + +def test_power_disposition_rejects_out_of_range_min_pass_rate() -> None: + from scripts.run_slm317_repair_hybrid import power_disposition + + for bad in (0.0, 1.0, -0.1, 1.5): + with pytest.raises(ValueError, match="min_pass_rate"): + power_disposition( + passed_seeds=1, + n_seeds=20, + min_pass_rate=bad, + safety_pass=True, + reachability_pass=True, + ) + + +def test_power_disposition_positive_at_boundary_clearing_ci() -> None: + """18/20 passes: Wilson 95% lower bound clears 0.5 -> repair_positive.""" + from scripts.run_slm317_repair_hybrid import power_disposition + + disposition, interval = power_disposition( + passed_seeds=18, + n_seeds=20, + min_pass_rate=0.5, + safety_pass=True, + reachability_pass=True, + ) + assert disposition == "repair_positive" + assert interval["low"] >= 0.5 + assert interval["n"] == 20 + + +def test_power_disposition_negative_when_pass_rate_ruled_out() -> None: + """0/20 passes: Wilson 95% upper bound falls below 0.5 -> repair_negative.""" + from scripts.run_slm317_repair_hybrid import power_disposition + + disposition, interval = power_disposition( + passed_seeds=0, + n_seeds=20, + min_pass_rate=0.5, + safety_pass=True, + reachability_pass=True, + ) + assert disposition == "repair_negative" + assert interval["high"] < 0.5 + + +def test_power_disposition_underpowered_when_interval_straddles() -> None: + """10/20 passes: the interval straddles 0.5 -> inconclusive_underpowered.""" + from scripts.run_slm317_repair_hybrid import power_disposition + + disposition, interval = power_disposition( + passed_seeds=10, + n_seeds=20, + min_pass_rate=0.5, + safety_pass=True, + reachability_pass=True, + ) + assert disposition == "inconclusive_underpowered" + assert interval["low"] < 0.5 <= interval["high"] + + +def test_power_disposition_safety_failure_is_negative_regardless() -> None: + """Safety gate failure forces repair_negative even at a high pass rate.""" + from scripts.run_slm317_repair_hybrid import power_disposition + + disposition, _ = power_disposition( + passed_seeds=20, + n_seeds=20, + min_pass_rate=0.5, + safety_pass=False, + reachability_pass=True, + ) + assert disposition == "repair_negative" + + +def test_power_disposition_reachability_failure_is_underpowered_not_positive() -> None: + """A clearing interval without the reachability gate cannot be positive.""" + from scripts.run_slm317_repair_hybrid import power_disposition + + disposition, _ = power_disposition( + passed_seeds=20, + n_seeds=20, + min_pass_rate=0.5, + safety_pass=True, + reachability_pass=False, + ) + assert disposition == "inconclusive_underpowered" + + +def test_power_rule_none_preserves_legacy_disposition_shape() -> None: + """power_rule=None must keep the exact original payload: no power keys, + legacy 'inconclusive' vocabulary, original issue id.""" + from scripts.run_slm317_repair_hybrid import render_markdown + + payload = json.loads( + Path("docs/design/iter-slm317-repair-hybrid-20260724.json").read_text() + ) + assert "power_rule" not in payload + assert "per_seed_results" not in payload + assert payload["issue"] == "SLM-317" + assert payload["disposition"] in { + "repair_positive", + "repair_negative", + "inconclusive", + } + # The legacy renderer must not emit a powered-rerun section. + assert "Preregistration (powered rerun" not in render_markdown(payload) + + +def test_main_rejects_nondefault_seeds_without_min_pass_rate() -> None: + from scripts.run_slm317_repair_hybrid import main + + with pytest.raises(SystemExit): + main(["--seeds", "2", "3"])